aboutsummaryrefslogtreecommitdiff
path: root/src/zencore/include
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2023-05-02 10:01:47 +0200
committerGitHub <[email protected]>2023-05-02 10:01:47 +0200
commit075d17f8ada47e990fe94606c3d21df409223465 (patch)
treee50549b766a2f3c354798a54ff73404217b4c9af /src/zencore/include
parentfix: bundle shouldn't append content zip to zen (diff)
downloadzen-075d17f8ada47e990fe94606c3d21df409223465.tar.xz
zen-075d17f8ada47e990fe94606c3d21df409223465.zip
moved source directories into `/src` (#264)
* moved source directories into `/src` * updated bundle.lua for new `src` path * moved some docs, icon * removed old test trees
Diffstat (limited to 'src/zencore/include')
-rw-r--r--src/zencore/include/zencore/atomic.h74
-rw-r--r--src/zencore/include/zencore/base64.h17
-rw-r--r--src/zencore/include/zencore/blake3.h62
-rw-r--r--src/zencore/include/zencore/blockingqueue.h76
-rw-r--r--src/zencore/include/zencore/compactbinary.h1475
-rw-r--r--src/zencore/include/zencore/compactbinarybuilder.h661
-rw-r--r--src/zencore/include/zencore/compactbinarypackage.h341
-rw-r--r--src/zencore/include/zencore/compactbinaryvalidation.h197
-rw-r--r--src/zencore/include/zencore/compactbinaryvalue.h290
-rw-r--r--src/zencore/include/zencore/compositebuffer.h142
-rw-r--r--src/zencore/include/zencore/compress.h165
-rw-r--r--src/zencore/include/zencore/config.h.in16
-rw-r--r--src/zencore/include/zencore/crc32.h13
-rw-r--r--src/zencore/include/zencore/crypto.h77
-rw-r--r--src/zencore/include/zencore/endian.h113
-rw-r--r--src/zencore/include/zencore/enumflags.h61
-rw-r--r--src/zencore/include/zencore/except.h57
-rw-r--r--src/zencore/include/zencore/filesystem.h190
-rw-r--r--src/zencore/include/zencore/fmtutils.h52
-rw-r--r--src/zencore/include/zencore/intmath.h183
-rw-r--r--src/zencore/include/zencore/iobuffer.h423
-rw-r--r--src/zencore/include/zencore/iohash.h115
-rw-r--r--src/zencore/include/zencore/logging.h136
-rw-r--r--src/zencore/include/zencore/md5.h50
-rw-r--r--src/zencore/include/zencore/memory.h401
-rw-r--r--src/zencore/include/zencore/meta.h30
-rw-r--r--src/zencore/include/zencore/mpscqueue.h110
-rw-r--r--src/zencore/include/zencore/refcount.h186
-rw-r--r--src/zencore/include/zencore/scopeguard.h45
-rw-r--r--src/zencore/include/zencore/session.h14
-rw-r--r--src/zencore/include/zencore/sha1.h76
-rw-r--r--src/zencore/include/zencore/sharedbuffer.h167
-rw-r--r--src/zencore/include/zencore/stats.h295
-rw-r--r--src/zencore/include/zencore/stream.h90
-rw-r--r--src/zencore/include/zencore/string.h1115
-rw-r--r--src/zencore/include/zencore/testing.h67
-rw-r--r--src/zencore/include/zencore/testutils.h32
-rw-r--r--src/zencore/include/zencore/thread.h273
-rw-r--r--src/zencore/include/zencore/timer.h58
-rw-r--r--src/zencore/include/zencore/trace.h36
-rw-r--r--src/zencore/include/zencore/uid.h87
-rw-r--r--src/zencore/include/zencore/varint.h277
-rw-r--r--src/zencore/include/zencore/windows.h25
-rw-r--r--src/zencore/include/zencore/workthreadpool.h48
-rw-r--r--src/zencore/include/zencore/xxhash.h89
-rw-r--r--src/zencore/include/zencore/zencore.h383
46 files changed, 8890 insertions, 0 deletions
diff --git a/src/zencore/include/zencore/atomic.h b/src/zencore/include/zencore/atomic.h
new file mode 100644
index 000000000..bf549e21d
--- /dev/null
+++ b/src/zencore/include/zencore/atomic.h
@@ -0,0 +1,74 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#if ZEN_COMPILER_MSC
+# include <intrin.h>
+#else
+# include <atomic>
+#endif
+
+#include <cinttypes>
+
+namespace zen {
+
+inline uint32_t
+AtomicIncrement(volatile uint32_t& value)
+{
+#if ZEN_COMPILER_MSC
+ return _InterlockedIncrement((long volatile*)&value);
+#else
+ return ((std::atomic<uint32_t>*)(&value))->fetch_add(1, std::memory_order_seq_cst) + 1;
+#endif
+}
+inline uint32_t
+AtomicDecrement(volatile uint32_t& value)
+{
+#if ZEN_COMPILER_MSC
+ return _InterlockedDecrement((long volatile*)&value);
+#else
+ return ((std::atomic<uint32_t>*)(&value))->fetch_sub(1, std::memory_order_seq_cst) - 1;
+#endif
+}
+
+inline uint64_t
+AtomicIncrement(volatile uint64_t& value)
+{
+#if ZEN_COMPILER_MSC
+ return _InterlockedIncrement64((__int64 volatile*)&value);
+#else
+ return ((std::atomic<uint64_t>*)(&value))->fetch_add(1, std::memory_order_seq_cst) + 1;
+#endif
+}
+inline uint64_t
+AtomicDecrement(volatile uint64_t& value)
+{
+#if ZEN_COMPILER_MSC
+ return _InterlockedDecrement64((__int64 volatile*)&value);
+#else
+ return ((std::atomic<uint64_t>*)(&value))->fetch_sub(1, std::memory_order_seq_cst) - 1;
+#endif
+}
+
+inline uint32_t
+AtomicAdd(volatile uint32_t& value, uint32_t amount)
+{
+#if ZEN_COMPILER_MSC
+ return _InterlockedExchangeAdd((long volatile*)&value, amount);
+#else
+ return ((std::atomic<uint32_t>*)(&value))->fetch_add(amount, std::memory_order_seq_cst);
+#endif
+}
+inline uint64_t
+AtomicAdd(volatile uint64_t& value, uint64_t amount)
+{
+#if ZEN_COMPILER_MSC
+ return _InterlockedExchangeAdd64((__int64 volatile*)&value, amount);
+#else
+ return ((std::atomic<uint64_t>*)(&value))->fetch_add(amount, std::memory_order_seq_cst);
+#endif
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/base64.h b/src/zencore/include/zencore/base64.h
new file mode 100644
index 000000000..4d78b085f
--- /dev/null
+++ b/src/zencore/include/zencore/base64.h
@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+struct Base64
+{
+ template<typename CharType>
+ static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest);
+
+ static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; }
+};
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/blake3.h b/src/zencore/include/zencore/blake3.h
new file mode 100644
index 000000000..b31b710a7
--- /dev/null
+++ b/src/zencore/include/zencore/blake3.h
@@ -0,0 +1,62 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <cinttypes>
+#include <compare>
+#include <cstring>
+
+#include <zencore/memory.h>
+
+namespace zen {
+
+class CompositeBuffer;
+class StringBuilderBase;
+
+/**
+ * BLAKE3 hash - 256 bits
+ */
+struct BLAKE3
+{
+ uint8_t Hash[32];
+
+ inline auto operator<=>(const BLAKE3& Rhs) const = default;
+
+ static BLAKE3 HashBuffer(const CompositeBuffer& Buffer);
+ static BLAKE3 HashMemory(const void* Data, size_t ByteCount);
+ static BLAKE3 FromHexString(const char* String);
+ const char* ToHexString(char* OutString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& OutBuilder) const;
+
+ static const int StringLength = 64;
+ typedef char String_t[StringLength + 1];
+
+ static BLAKE3 Zero; // Initialized to all zeroes
+
+ struct Hasher
+ {
+ size_t operator()(const BLAKE3& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct BLAKE3Stream
+{
+ BLAKE3Stream();
+
+ void Reset(); // Begin streaming hash compute (not needed on freshly constructed instance)
+ BLAKE3Stream& Append(const void* data, size_t byteCount); // Append another chunk
+ BLAKE3Stream& Append(MemoryView DataView) { return Append(DataView.GetData(), DataView.GetSize()); } // Append another chunk
+ BLAKE3 GetHash(); // Obtain final hash. If you wish to reuse the instance call reset()
+
+private:
+ alignas(16) uint8_t m_HashState[2048];
+};
+
+void blake3_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/blockingqueue.h b/src/zencore/include/zencore/blockingqueue.h
new file mode 100644
index 000000000..f92df5a54
--- /dev/null
+++ b/src/zencore/include/zencore/blockingqueue.h
@@ -0,0 +1,76 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <atomic>
+#include <condition_variable>
+#include <deque>
+#include <mutex>
+
+namespace zen {
+
+template<typename T>
+class BlockingQueue
+{
+public:
+ BlockingQueue() = default;
+
+ ~BlockingQueue() { CompleteAdding(); }
+
+ void Enqueue(T&& Item)
+ {
+ {
+ std::lock_guard Lock(m_Lock);
+ m_Queue.emplace_back(std::move(Item));
+ m_Size++;
+ }
+
+ m_NewItemSignal.notify_one();
+ }
+
+ bool WaitAndDequeue(T& Item)
+ {
+ if (m_CompleteAdding.load())
+ {
+ return false;
+ }
+
+ std::unique_lock Lock(m_Lock);
+ m_NewItemSignal.wait(Lock, [this]() { return !m_Queue.empty() || m_CompleteAdding.load(); });
+
+ if (!m_Queue.empty())
+ {
+ Item = std::move(m_Queue.front());
+ m_Queue.pop_front();
+ m_Size--;
+
+ return true;
+ }
+
+ return false;
+ }
+
+ void CompleteAdding()
+ {
+ if (!m_CompleteAdding.load())
+ {
+ m_CompleteAdding.store(true);
+ m_NewItemSignal.notify_all();
+ }
+ }
+
+ std::size_t Size() const
+ {
+ std::unique_lock Lock(m_Lock);
+ return m_Queue.size();
+ }
+
+private:
+ mutable std::mutex m_Lock;
+ std::condition_variable m_NewItemSignal;
+ std::deque<T> m_Queue;
+ std::atomic_bool m_CompleteAdding{false};
+ std::atomic_uint32_t m_Size;
+};
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h
new file mode 100644
index 000000000..b546f97aa
--- /dev/null
+++ b/src/zencore/include/zencore/compactbinary.h
@@ -0,0 +1,1475 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/enumflags.h>
+#include <zencore/intmath.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/memory.h>
+#include <zencore/meta.h>
+#include <zencore/sharedbuffer.h>
+#include <zencore/uid.h>
+#include <zencore/varint.h>
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class CbObjectView;
+class CbArrayView;
+class BinaryReader;
+class BinaryWriter;
+class CompressedBuffer;
+class CbValue;
+
+class DateTime
+{
+public:
+ explicit DateTime(uint64_t InTicks) : Ticks(InTicks) {}
+ inline DateTime(int Year, int Month, int Day, int Hours = 0, int Minutes = 0, int Seconds = 0, int MilliSeconds = 0)
+ {
+ Set(Year, Month, Day, Hours, Minutes, Seconds, MilliSeconds);
+ }
+
+ inline uint64_t GetTicks() const { return Ticks; }
+
+ static uint64_t NowTicks();
+ static DateTime Now();
+
+ int GetYear() const;
+ int GetMonth() const;
+ int GetDay() const;
+ int GetHour() const;
+ int GetHour12() const;
+ int GetMinute() const;
+ int GetSecond() const;
+ int GetMillisecond() const;
+ void GetDate(int& Year, int& Month, int& Day) const;
+
+ inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
+ inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+ std::string ToString(const char* Format) const;
+ std::string ToIso8601() const;
+
+private:
+ void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond);
+ uint64_t Ticks; // 1 tick == 0.1us == 100ns, epoch == Jan 1st 0001
+};
+
+class TimeSpan
+{
+public:
+ explicit TimeSpan(uint64_t InTicks) : Ticks(InTicks) {}
+ inline TimeSpan(int Hours, int Minutes, int Seconds) { Set(0, Hours, Minutes, Seconds, 0); }
+ inline TimeSpan(int Days, int Hours, int Minutes, int Seconds) { Set(Days, Hours, Minutes, Seconds, 0); }
+ inline TimeSpan(int Days, int Hours, int Minutes, int Seconds, int Nanos) { Set(Days, Hours, Minutes, Seconds, Nanos); }
+
+ inline uint64_t GetTicks() const { return Ticks; }
+ inline bool operator==(const TimeSpan& Rhs) const { return Ticks == Rhs.Ticks; }
+ inline auto operator<=>(const TimeSpan& Rhs) const { return Ticks - Rhs.Ticks; }
+
+ /**
+ * Time span related constants.
+ */
+
+ /** The maximum number of ticks that can be represented in FTimespan. */
+ static constexpr int64_t MaxTicks = 9223372036854775807;
+
+ /** The minimum number of ticks that can be represented in FTimespan. */
+ static constexpr int64_t MinTicks = -9223372036854775807 - 1;
+
+ /** The number of nanoseconds per tick. */
+ static constexpr int64_t NanosecondsPerTick = 100;
+
+ /** The number of timespan ticks per day. */
+ static constexpr int64_t TicksPerDay = 864000000000;
+
+ /** The number of timespan ticks per hour. */
+ static constexpr int64_t TicksPerHour = 36000000000;
+
+ /** The number of timespan ticks per microsecond. */
+ static constexpr int64_t TicksPerMicrosecond = 10;
+
+ /** The number of timespan ticks per millisecond. */
+ static constexpr int64_t TicksPerMillisecond = 10000;
+
+ /** The number of timespan ticks per minute. */
+ static constexpr int64_t TicksPerMinute = 600000000;
+
+ /** The number of timespan ticks per second. */
+ static constexpr int64_t TicksPerSecond = 10000000;
+
+ /** The number of timespan ticks per week. */
+ static constexpr int64_t TicksPerWeek = 6048000000000;
+
+ /** The number of timespan ticks per year (365 days, not accounting for leap years). */
+ static constexpr int64_t TicksPerYear = 365 * TicksPerDay;
+
+ int GetFractionTicks() const { return (int)(Ticks % TicksPerSecond); }
+
+ int GetFractionMicro() const { return (int)((Ticks % TicksPerSecond) / TicksPerMicrosecond); }
+
+ int GetFractionMilli() const { return (int)((Ticks % TicksPerSecond) / TicksPerMillisecond); }
+
+ int GetFractionNano() const { return (int)((Ticks % TicksPerSecond) * NanosecondsPerTick); }
+
+ int GetDays() const { return (int)(Ticks / TicksPerDay); }
+
+ int GetHours() const { return (int)((Ticks / TicksPerHour) % 24); }
+
+ int GetMinutes() const { return (int)((Ticks / TicksPerMinute) % 60); }
+
+ int GetSeconds() const { return (int)((Ticks / TicksPerSecond) % 60); }
+
+ ZENCORE_API std::string ToString(const char* Format) const;
+ ZENCORE_API std::string ToString() const;
+
+private:
+ void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano);
+
+ uint64_t Ticks;
+};
+
+struct Guid
+{
+ uint32_t A, B, C, D;
+
+ StringBuilderBase& ToString(StringBuilderBase& OutString) const;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+/**
+ * Field types and flags for CbField.
+ *
+ * This is a private type and is only declared here to enable inline use below.
+ *
+ * DO NOT CHANGE THE VALUE OF ANY MEMBERS OF THIS ENUM!
+ * BACKWARD COMPATIBILITY REQUIRES THAT THESE VALUES BE FIXED!
+ * SERIALIZATION USES HARD-CODED CONSTANTS BASED ON THESE VALUES!
+ */
+enum class CbFieldType : uint8_t
+{
+ /** A field type that does not occur in a valid object. */
+ None = 0x00,
+
+ /** Null. Payload is empty. */
+ Null = 0x01,
+
+ /**
+ * Object is an array of fields with unique non-empty names.
+ *
+ * Payload is a VarUInt byte count for the encoded fields followed by the fields.
+ */
+ Object = 0x02,
+ /**
+ * UniformObject is an array of fields with the same field types and unique non-empty names.
+ *
+ * Payload is a VarUInt byte count for the encoded fields followed by the fields.
+ */
+ UniformObject = 0x03,
+
+ /**
+ * Array is an array of fields with no name that may be of different types.
+ *
+ * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by the fields.
+ */
+ Array = 0x04,
+ /**
+ * UniformArray is an array of fields with no name and with the same field type.
+ *
+ * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by field type,
+ * followed by the fields without their field type.
+ */
+ UniformArray = 0x05,
+
+ /** Binary. Payload is a VarUInt byte count followed by the data. */
+ Binary = 0x06,
+
+ /** String in UTF-8. Payload is a VarUInt byte count then an unterminated UTF-8 string. */
+ String = 0x07,
+
+ /**
+ * Non-negative integer with the range of a 64-bit unsigned integer.
+ *
+ * Payload is the value encoded as a VarUInt.
+ */
+ IntegerPositive = 0x08,
+ /**
+ * Negative integer with the range of a 64-bit signed integer.
+ *
+ * Payload is the ones' complement of the value encoded as a VarUInt.
+ */
+ IntegerNegative = 0x09,
+
+ /** Single precision float. Payload is one big endian IEEE 754 binary32 float. */
+ Float32 = 0x0a,
+ /** Double precision float. Payload is one big endian IEEE 754 binary64 float. */
+ Float64 = 0x0b,
+
+ /** Boolean false value. Payload is empty. */
+ BoolFalse = 0x0c,
+ /** Boolean true value. Payload is empty. */
+ BoolTrue = 0x0d,
+
+ /**
+ * ObjectAttachment is a reference to a compact binary attachment stored externally.
+ *
+ * Payload is a 160-bit hash digest of the referenced compact binary data.
+ */
+ ObjectAttachment = 0x0e,
+ /**
+ * BinaryAttachment is a reference to a binary attachment stored externally.
+ *
+ * Payload is a 160-bit hash digest of the referenced binary data.
+ */
+ BinaryAttachment = 0x0f,
+
+ /** Hash. Payload is a 160-bit hash digest. */
+ Hash = 0x10,
+ /** UUID/GUID. Payload is a 128-bit UUID as defined by RFC 4122. */
+ Uuid = 0x11,
+
+ /**
+ * Date and time between 0001-01-01 00:00:00.0000000 and 9999-12-31 23:59:59.9999999.
+ *
+ * Payload is a big endian int64 count of 100ns ticks since 0001-01-01 00:00:00.0000000.
+ */
+ DateTime = 0x12,
+ /**
+ * Difference between two date/time values.
+ *
+ * Payload is a big endian int64 count of 100ns ticks in the span, and may be negative.
+ */
+ TimeSpan = 0x13,
+
+ /**
+ * Object ID
+ *
+ * Payload is a 12-byte opaque identifier
+ */
+ ObjectId = 0x14,
+
+ /**
+ * CustomById identifies the sub-type of its payload by an integer identifier.
+ *
+ * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed
+ * by a VarUInt of the sub-type identifier then the payload of the sub-type.
+ */
+ CustomById = 0x1e,
+ /**
+ * CustomByType identifies the sub-type of its payload by a string identifier.
+ *
+ * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed
+ * by a VarUInt byte count of the unterminated sub-type identifier, then the sub-type identifier
+ * without termination, then the payload of the sub-type.
+ */
+ CustomByName = 0x1f,
+
+ /** Reserved for future use as a flag. Do not add types in this range. */
+ Reserved = 0x20,
+
+ /**
+ * A transient flag which indicates that the object or array containing this field has stored
+ * the field type before the payload and name. Non-uniform objects and fields will set this.
+ *
+ * Note: Since the flag must never be serialized, this bit may be repurposed in the future.
+ */
+ HasFieldType = 0x40,
+
+ /** A persisted flag which indicates that the field has a name stored before the payload. */
+ HasFieldName = 0x80,
+};
+
+ENUM_CLASS_FLAGS(CbFieldType);
+
+/** Functions that operate on CbFieldType. */
+class CbFieldTypeOps
+{
+ static constexpr CbFieldType SerializedTypeMask = CbFieldType(0b1011'1111);
+ static constexpr CbFieldType TypeMask = CbFieldType(0b0011'1111);
+ static constexpr CbFieldType ObjectMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType ObjectBase = CbFieldType(0b0000'0010);
+ static constexpr CbFieldType ArrayMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType ArrayBase = CbFieldType(0b0000'0100);
+ static constexpr CbFieldType IntegerMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType IntegerBase = CbFieldType(0b0000'1000);
+ static constexpr CbFieldType FloatMask = CbFieldType(0b0011'1100);
+ static constexpr CbFieldType FloatBase = CbFieldType(0b0000'1000);
+ static constexpr CbFieldType BoolMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType BoolBase = CbFieldType(0b0000'1100);
+ static constexpr CbFieldType AttachmentMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType AttachmentBase = CbFieldType(0b0000'1110);
+
+ static void StaticAssertTypeConstants();
+
+public:
+ /** The type with flags removed. */
+ static constexpr inline CbFieldType GetType(CbFieldType Type) { return Type & TypeMask; }
+ /** The type with transient flags removed. */
+ static constexpr inline CbFieldType GetSerializedType(CbFieldType Type) { return Type & SerializedTypeMask; }
+
+ static constexpr inline bool HasFieldType(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldType); }
+ static constexpr inline bool HasFieldName(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldName); }
+
+ static constexpr inline bool IsNone(CbFieldType Type) { return GetType(Type) == CbFieldType::None; }
+ static constexpr inline bool IsNull(CbFieldType Type) { return GetType(Type) == CbFieldType::Null; }
+
+ static constexpr inline bool IsObject(CbFieldType Type) { return (Type & ObjectMask) == ObjectBase; }
+ static constexpr inline bool IsArray(CbFieldType Type) { return (Type & ArrayMask) == ArrayBase; }
+
+ static constexpr inline bool IsBinary(CbFieldType Type) { return GetType(Type) == CbFieldType::Binary; }
+ static constexpr inline bool IsString(CbFieldType Type) { return GetType(Type) == CbFieldType::String; }
+
+ static constexpr inline bool IsInteger(CbFieldType Type) { return (Type & IntegerMask) == IntegerBase; }
+ /** Whether the field is a float, or integer due to implicit conversion. */
+ static constexpr inline bool IsFloat(CbFieldType Type) { return (Type & FloatMask) == FloatBase; }
+ static constexpr inline bool IsBool(CbFieldType Type) { return (Type & BoolMask) == BoolBase; }
+
+ static constexpr inline bool IsObjectAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectAttachment; }
+ static constexpr inline bool IsBinaryAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::BinaryAttachment; }
+ static constexpr inline bool IsAttachment(CbFieldType Type) { return (Type & AttachmentMask) == AttachmentBase; }
+
+ static constexpr inline bool IsHash(CbFieldType Type)
+ {
+ switch (GetType(Type))
+ {
+ case CbFieldType::Hash:
+ case CbFieldType::BinaryAttachment:
+ case CbFieldType::ObjectAttachment:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; }
+ static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; }
+
+ static constexpr inline bool IsCustomById(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomById; }
+ static constexpr inline bool IsCustomByName(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomByName; }
+
+ static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; }
+ static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; }
+
+ /** Whether the type is or may contain fields of any attachment type. */
+ static constexpr inline bool MayContainAttachments(CbFieldType Type)
+ {
+ return int(IsObject(Type) == true) | int(IsArray(Type) == true) | int(IsAttachment(Type) == true);
+ }
+};
+
+/** Errors that can occur when accessing a field. */
+enum class CbFieldError : uint8_t
+{
+ /** The field is not in an error state. */
+ None,
+ /** The value type does not match the requested type. */
+ TypeError,
+ /** The value is out of range for the requested type. */
+ RangeError,
+};
+
+class ICbVisitor
+{
+public:
+ virtual void SetName(std::string_view Name) = 0;
+ virtual void BeginObject() = 0;
+ virtual void EndObject() = 0;
+ virtual void BeginArray() = 0;
+ virtual void EndArray() = 0;
+ virtual void VisitNull() = 0;
+ virtual void VisitBinary(SharedBuffer Value) = 0;
+ virtual void VisitString(std::string_view Value) = 0;
+ virtual void VisitInteger(int64_t Value) = 0;
+ virtual void VisitInteger(uint64_t Value) = 0;
+ virtual void VisitFloat(float Value) = 0;
+ virtual void VisitDouble(double Value) = 0;
+ virtual void VisitBool(bool value) = 0;
+ virtual void VisitCbAttachment(const IoHash& Value) = 0;
+ virtual void VisitBinaryAttachment(const IoHash& Value) = 0;
+ virtual void VisitHash(const IoHash& Value) = 0;
+ virtual void VisitUuid(const Guid& Value) = 0;
+ virtual void VisitObjectId(const Oid& Value) = 0;
+ virtual void VisitDateTime(DateTime Value) = 0;
+ virtual void VisitTimeSpan(TimeSpan Value) = 0;
+};
+
+/** A custom compact binary field type with an integer identifier. */
+struct CbCustomById
+{
+ /** An identifier for the sub-type of the field. */
+ uint64_t Id = 0;
+ /** A view of the value. Lifetime is tied to the field that the value is associated with. */
+ MemoryView Data;
+};
+
+/** A custom compact binary field type with a string identifier. */
+struct CbCustomByName
+{
+ /** An identifier for the sub-type of the field. Lifetime is tied to the field that the name is associated with. */
+ std::u8string_view Name;
+ /** A view of the value. Lifetime is tied to the field that the value is associated with. */
+ MemoryView Data;
+};
+
+namespace CompactBinaryPrivate {
+ /** Parameters for converting to an integer. */
+ struct IntegerParams
+ {
+ /** Whether the output type has a sign bit. */
+ uint32_t IsSigned : 1;
+ /** Bits of magnitude. (7 for int8) */
+ uint32_t MagnitudeBits : 31;
+ };
+
+ /** Make integer params for the given integer type. */
+ template<typename IntType>
+ static constexpr inline IntegerParams MakeIntegerParams()
+ {
+ IntegerParams Params;
+ Params.IsSigned = IntType(-1) < IntType(0);
+ Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned;
+ return Params;
+ }
+
+} // namespace CompactBinaryPrivate
+
+/**
+ * An atom of data in the compact binary format.
+ *
+ * Accessing the value of a field is always a safe operation, even if accessed as the wrong type.
+ * An invalid access will return a default value for the requested type, and set an error code on
+ * the field that can be checked with GetLastError and HasLastError. A valid access will clear an
+ * error from a previous invalid access.
+ *
+ * A field is encoded in one or more bytes, depending on its type and the type of object or array
+ * that contains it. A field of an object or array which is non-uniform encodes its field type in
+ * the first byte, and includes the HasFieldName flag for a field in an object. The field name is
+ * encoded in a variable-length unsigned integer of its size in bytes, for named fields, followed
+ * by that many bytes of the UTF-8 encoding of the name with no null terminator. The remainder of
+ * the field is the payload and is described in the field type enum. Every field must be uniquely
+ * addressable when encoded, which means a zero-byte field is not permitted, and only arises in a
+ * uniform array of fields with no payload, where the answer is to encode as a non-uniform array.
+ *
+ * This type only provides a view into memory and does not perform any memory management itself.
+ * Use CbFieldRef to hold a reference to the underlying memory when necessary.
+ */
+
+class CbFieldView
+{
+public:
+ CbFieldView() = default;
+
+ ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType);
+
+ /** Construct a field from a value, without access to the name. */
+ inline explicit CbFieldView(const CbValue& Value);
+
+ /** Returns the name of the field if it has a name, otherwise an empty view. */
+ constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); }
+ /** Returns the name of the field if it has a name, otherwise an empty view. */
+ constexpr inline std::u8string_view GetU8Name() const
+ {
+ return std::u8string_view(static_cast<const char8_t*>(Payload) - NameLen, NameLen);
+ }
+
+ /** Returns the value for unchecked access. Prefer the typed accessors below. */
+ inline CbValue GetValue() const;
+
+ ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView());
+ ZENCORE_API CbObjectView AsObjectView();
+ ZENCORE_API CbArrayView AsArrayView();
+ ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view());
+ ZENCORE_API std::u8string_view AsU8String(std::u8string_view Default = std::u8string_view());
+
+ ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const;
+
+ /** Access the field as an int8. Returns the provided default on error. */
+ inline int8_t AsInt8(int8_t Default = 0) { return AsInteger<int8_t>(Default); }
+ /** Access the field as an int16. Returns the provided default on error. */
+ inline int16_t AsInt16(int16_t Default = 0) { return AsInteger<int16_t>(Default); }
+ /** Access the field as an int32. Returns the provided default on error. */
+ inline int32_t AsInt32(int32_t Default = 0) { return AsInteger<int32_t>(Default); }
+ /** Access the field as an int64. Returns the provided default on error. */
+ inline int64_t AsInt64(int64_t Default = 0) { return AsInteger<int64_t>(Default); }
+ /** Access the field as a uint8. Returns the provided default on error. */
+ inline uint8_t AsUInt8(uint8_t Default = 0) { return AsInteger<uint8_t>(Default); }
+ /** Access the field as a uint16. Returns the provided default on error. */
+ inline uint16_t AsUInt16(uint16_t Default = 0) { return AsInteger<uint16_t>(Default); }
+ /** Access the field as a uint32. Returns the provided default on error. */
+ inline uint32_t AsUInt32(uint32_t Default = 0) { return AsInteger<uint32_t>(Default); }
+ /** Access the field as a uint64. Returns the provided default on error. */
+ inline uint64_t AsUInt64(uint64_t Default = 0) { return AsInteger<uint64_t>(Default); }
+
+ /** Access the field as a float. Returns the provided default on error. */
+ ZENCORE_API float AsFloat(float Default = 0.0f);
+ /** Access the field as a double. Returns the provided default on error. */
+ ZENCORE_API double AsDouble(double Default = 0.0);
+
+ /** Access the field as a bool. Returns the provided default on error. */
+ ZENCORE_API bool AsBool(bool bDefault = false);
+
+ /** Access the field as a hash referencing a compact binary attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsObjectAttachment(const IoHash& Default = IoHash());
+ /** Access the field as a hash referencing a binary attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsBinaryAttachment(const IoHash& Default = IoHash());
+ /** Access the field as a hash referencing an attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsAttachment(const IoHash& Default = IoHash());
+
+ /** Access the field as a hash. Returns the provided default on error. */
+ ZENCORE_API IoHash AsHash(const IoHash& Default = IoHash());
+
+ /** Access the field as a UUID. Returns a nil UUID on error. */
+ ZENCORE_API Guid AsUuid();
+ /** Access the field as a UUID. Returns the provided default on error. */
+ ZENCORE_API Guid AsUuid(const Guid& Default);
+
+ /** Access the field as an OID. Returns a nil OID on error. */
+ ZENCORE_API Oid AsObjectId();
+ /** Access the field as a OID. Returns the provided default on error. */
+ ZENCORE_API Oid AsObjectId(const Oid& Default);
+
+ /** Access the field as a custom sub-type with an integer identifier. Returns the provided default on error. */
+ ZENCORE_API CbCustomById AsCustomById(CbCustomById Default);
+ /** Access the field as a custom sub-type with a string identifier. Returns the provided default on error. */
+ ZENCORE_API CbCustomByName AsCustomByName(CbCustomByName Default);
+
+ /** Access the field as a date/time tick count. Returns the provided default on error. */
+ ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0);
+
+ /** Access the field as a date/time. Returns a date/time at the epoch on error. */
+ ZENCORE_API DateTime AsDateTime();
+ /** Access the field as a date/time. Returns the provided default on error. */
+ ZENCORE_API DateTime AsDateTime(DateTime Default);
+
+ /** Access the field as a timespan tick count. Returns the provided default on error. */
+ ZENCORE_API int64_t AsTimeSpanTicks(int64_t Default = 0);
+
+ /** Access the field as a timespan. Returns an empty timespan on error. */
+ ZENCORE_API TimeSpan AsTimeSpan();
+ /** Access the field as a timespan. Returns the provided default on error. */
+ ZENCORE_API TimeSpan AsTimeSpan(TimeSpan Default);
+
+ /** True if the field has a name. */
+ constexpr inline bool HasName() const { return CbFieldTypeOps::HasFieldName(Type); }
+
+ constexpr inline bool IsNull() const { return CbFieldTypeOps::IsNull(Type); }
+
+ constexpr inline bool IsObject() const { return CbFieldTypeOps::IsObject(Type); }
+ constexpr inline bool IsArray() const { return CbFieldTypeOps::IsArray(Type); }
+
+ constexpr inline bool IsBinary() const { return CbFieldTypeOps::IsBinary(Type); }
+ constexpr inline bool IsString() const { return CbFieldTypeOps::IsString(Type); }
+
+ /** Whether the field is an integer of unspecified range and sign. */
+ constexpr inline bool IsInteger() const { return CbFieldTypeOps::IsInteger(Type); }
+ /** Whether the field is a float, or integer that supports implicit conversion. */
+ constexpr inline bool IsFloat() const { return CbFieldTypeOps::IsFloat(Type); }
+ constexpr inline bool IsBool() const { return CbFieldTypeOps::IsBool(Type); }
+
+ constexpr inline bool IsObjectAttachment() const { return CbFieldTypeOps::IsObjectAttachment(Type); }
+ constexpr inline bool IsBinaryAttachment() const { return CbFieldTypeOps::IsBinaryAttachment(Type); }
+ constexpr inline bool IsAttachment() const { return CbFieldTypeOps::IsAttachment(Type); }
+
+ constexpr inline bool IsHash() const { return CbFieldTypeOps::IsHash(Type); }
+ constexpr inline bool IsUuid() const { return CbFieldTypeOps::IsUuid(Type); }
+ constexpr inline bool IsObjectId() const { return CbFieldTypeOps::IsObjectId(Type); }
+
+ constexpr inline bool IsDateTime() const { return CbFieldTypeOps::IsDateTime(Type); }
+ constexpr inline bool IsTimeSpan() const { return CbFieldTypeOps::IsTimeSpan(Type); }
+
+ /** Whether the field has a value. */
+ constexpr inline explicit operator bool() const { return HasValue(); }
+
+ /**
+ * Whether the field has a value.
+ *
+ * All fields in a valid object or array have a value. A field with no value is returned when
+ * finding a field by name fails or when accessing an iterator past the end.
+ */
+ constexpr inline bool HasValue() const { return !CbFieldTypeOps::IsNone(Type); };
+
+ /** Whether the last field access encountered an error. */
+ constexpr inline bool HasError() const { return Error != CbFieldError::None; }
+
+ /** The type of error that occurred on the last field access, or None. */
+ constexpr inline CbFieldError GetError() const { return Error; }
+
+ /** Returns the size of the field in bytes, including the type and name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the field, including the type and name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& HashStream) const;
+
+ /** Feed the field (including type and name) to the stream function */
+ inline void WriteToStream(auto Hash) const
+ {
+ const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type);
+ Hash(&SerializedType, sizeof(SerializedType));
+ auto View = GetViewNoType();
+ Hash(View.GetData(), View.GetSize());
+ }
+
+ /** Copy the field into a buffer of exactly GetSize() bytes, including the type and name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the field into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ /**
+ * Whether this field is identical to the other field.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be performed with ValidateCompactBinary, except for field order and field name case.
+ */
+ ZENCORE_API bool Equals(const CbFieldView& Other) const;
+
+ /** Returns a view of the field, including the type and name when present. */
+ ZENCORE_API MemoryView GetView() const;
+
+ /**
+ * Try to get a view of the field as it would be serialized, such as by CopyTo.
+ *
+ * A serialized view is not available if the field has an externally-provided type.
+ * Access the serialized form of such fields using CopyTo or FCbFieldRef::Clone.
+ */
+ inline bool TryGetSerializedView(MemoryView& OutView) const
+ {
+ if (CbFieldTypeOps::HasFieldType(Type))
+ {
+ OutView = GetView();
+ return true;
+ }
+ return false;
+ }
+
+protected:
+ /** Returns a view of the name and value payload, which excludes the type. */
+ ZENCORE_API MemoryView GetViewNoType() const;
+
+ /** Returns a view of the value payload, which excludes the type and name. */
+ inline MemoryView GetPayloadView() const { return MemoryView(Payload, GetPayloadSize()); }
+
+ /** Returns the type of the field including flags. */
+ constexpr inline CbFieldType GetType() const { return Type; }
+
+ /** Returns the start of the value payload. */
+ constexpr inline const void* GetPayload() const { return Payload; }
+
+ /** Returns the end of the value payload. */
+ inline const void* GetPayloadEnd() const { return static_cast<const uint8_t*>(Payload) + GetPayloadSize(); }
+
+ /** Returns the size of the value payload in bytes, which is the field excluding the type and name. */
+ ZENCORE_API uint64_t GetPayloadSize() const;
+
+ /** Assign a field from a pointer to its data and an optional externally-provided type. */
+ inline void Assign(const void* InData, const CbFieldType InType)
+ {
+ static_assert(std::is_trivially_destructible<CbFieldView>::value,
+ "This optimization requires CbField to be trivially destructible!");
+ new (this) CbFieldView(InData, InType);
+ }
+
+private:
+ /**
+ * Access the field as the given integer type.
+ *
+ * Returns the provided default if the value cannot be represented in the output type.
+ */
+ template<typename IntType>
+ inline IntType AsInteger(IntType Default)
+ {
+ return IntType(AsInteger(uint64_t(Default), CompactBinaryPrivate::MakeIntegerParams<IntType>()));
+ }
+
+ ZENCORE_API uint64_t AsInteger(uint64_t Default, CompactBinaryPrivate::IntegerParams Params);
+
+private:
+ /** The field type, with the transient HasFieldType flag if the field contains its type. */
+ CbFieldType Type = CbFieldType::None;
+ /** The error (if any) that occurred on the last field access. */
+ CbFieldError Error = CbFieldError::None;
+ /** The number of bytes for the name stored before the payload. */
+ uint32_t NameLen = 0;
+ /** The value payload, which also points to the end of the name. */
+ const void* Payload = nullptr;
+};
+
+template<typename FieldType>
+class TCbFieldIterator : public FieldType
+{
+public:
+ /** Construct an empty field range. */
+ constexpr TCbFieldIterator() = default;
+
+ inline TCbFieldIterator& operator++()
+ {
+ const void* const PayloadEnd = FieldType::GetPayloadEnd();
+ const int64_t AtEndMask = int64_t(PayloadEnd == FieldsEnd) - 1;
+ const CbFieldType NextType = CbFieldType(int64_t(FieldType::GetType()) & AtEndMask);
+ const void* const NextField = reinterpret_cast<const void*>(int64_t(PayloadEnd) & AtEndMask);
+ const void* const NextFieldsEnd = reinterpret_cast<const void*>(int64_t(FieldsEnd) & AtEndMask);
+
+ FieldType::Assign(NextField, NextType);
+ FieldsEnd = NextFieldsEnd;
+ return *this;
+ }
+
+ inline TCbFieldIterator operator++(int)
+ {
+ TCbFieldIterator It(*this);
+ ++*this;
+ return It;
+ }
+
+ constexpr inline FieldType& operator*() { return *this; }
+ constexpr inline FieldType* operator->() { return this; }
+
+ /** Reset this to an empty field range. */
+ inline void Reset() { *this = TCbFieldIterator(); }
+
+ /** Returns the size of the fields in the range in bytes. */
+ ZENCORE_API uint64_t GetRangeSize() const;
+
+ /** Calculate the hash of every field in the range. */
+ ZENCORE_API IoHash GetRangeHash() const;
+ ZENCORE_API void GetRangeHash(IoHashStream& Hash) const;
+
+ using FieldType::Equals;
+
+ template<typename OtherFieldType>
+ constexpr inline bool Equals(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return FieldType::GetPayload() == Other.OtherFieldType::GetPayload() && FieldsEnd == Other.FieldsEnd;
+ }
+
+ template<typename OtherFieldType>
+ constexpr inline bool operator==(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return Equals(Other);
+ }
+
+ template<typename OtherFieldType>
+ constexpr inline bool operator!=(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return !Equals(Other);
+ }
+
+ /** Copy the field range into a buffer of exactly GetRangeSize() bytes. */
+ ZENCORE_API void CopyRangeTo(MutableMemoryView Buffer) const;
+
+ /** Invoke the visitor for every attachment in the field range. */
+ ZENCORE_API void IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const;
+
+ /** Create a view of every field in the range. */
+ inline MemoryView GetRangeView() const { return MemoryView(FieldType::GetView().GetData(), FieldsEnd); }
+
+ /**
+ * Try to get a view of every field in the range as they would be serialized.
+ *
+ * A serialized view is not available if the underlying fields have an externally-provided type.
+ * Access the serialized form of such ranges using CbFieldRefIterator::CloneRange.
+ */
+ inline bool TryGetSerializedRangeView(MemoryView& OutView) const
+ {
+ if (CbFieldTypeOps::HasFieldType(FieldType::GetType()))
+ {
+ OutView = GetRangeView();
+ return true;
+ }
+ return false;
+ }
+
+protected:
+ /** Construct a field range that contains exactly one field. */
+ constexpr inline explicit TCbFieldIterator(FieldType InField) : FieldType(std::move(InField)), FieldsEnd(FieldType::GetPayloadEnd()) {}
+
+ /**
+ * Construct a field range from the first field and a pointer to the end of the last field.
+ *
+ * @param InField The first field, or the default field if there are no fields.
+ * @param InFieldsEnd A pointer to the end of the payload of the last field, or null.
+ */
+ constexpr inline TCbFieldIterator(FieldType&& InField, const void* InFieldsEnd) : FieldType(std::move(InField)), FieldsEnd(InFieldsEnd)
+ {
+ }
+
+ /** Returns the end of the last field, or null for an iterator at the end. */
+ template<typename OtherFieldType>
+ static inline const void* GetFieldsEnd(const TCbFieldIterator<OtherFieldType>& It)
+ {
+ return It.FieldsEnd;
+ }
+
+private:
+ friend inline TCbFieldIterator begin(const TCbFieldIterator& Iterator) { return Iterator; }
+ friend inline TCbFieldIterator end(const TCbFieldIterator&) { return TCbFieldIterator(); }
+
+private:
+ template<typename OtherType>
+ friend class TCbFieldIterator;
+
+ friend class CbFieldViewIterator;
+
+ friend class CbFieldIterator;
+
+ /** Pointer to the first byte past the end of the last field. Set to null at the end. */
+ const void* FieldsEnd = nullptr;
+};
+
+/**
+ * Iterator for CbField.
+ *
+ * @see CbFieldIterator
+ */
+class CbFieldViewIterator : public TCbFieldIterator<CbFieldView>
+{
+public:
+ constexpr CbFieldViewIterator() = default;
+
+ /** Construct a field range that contains exactly one field. */
+ static inline CbFieldViewIterator MakeSingle(const CbFieldView& Field) { return CbFieldViewIterator(Field); }
+
+ /**
+ * Construct a field range from a buffer containing zero or more valid fields.
+ *
+ * @param View A buffer containing zero or more valid fields.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ */
+ static inline CbFieldViewIterator MakeRange(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ return !View.IsEmpty() ? TCbFieldIterator(CbFieldView(View.GetData(), Type), View.GetDataEnd()) : CbFieldViewIterator();
+ }
+
+ /** Construct an iterator from another iterator. */
+ template<typename OtherFieldType>
+ inline CbFieldViewIterator(const TCbFieldIterator<OtherFieldType>& It)
+ : TCbFieldIterator(ImplicitConv<CbFieldView>(It), GetFieldsEnd(It))
+ {
+ }
+
+private:
+ using TCbFieldIterator::TCbFieldIterator;
+};
+
+/**
+ * Serialize a compact binary array to JSON.
+ */
+ZENCORE_API void CompactBinaryToJson(const CbArrayView& Object, StringBuilderBase& Builder);
+
+/**
+ * Array of CbField that have no names.
+ *
+ * Accessing a field of the array requires iteration. Access by index is not provided because the
+ * cost of accessing an item by index scales linearly with the index.
+ *
+ * This type only provides a view into memory and does not perform any memory management itself.
+ * Use CbArrayRef to hold a reference to the underlying memory when necessary.
+ */
+class CbArrayView : protected CbFieldView
+{
+ friend class CbFieldView;
+
+public:
+ /** @see CbField::CbField */
+ using CbFieldView::CbFieldView;
+
+ /** Construct an array with no fields. */
+ ZENCORE_API CbArrayView();
+
+ /** Returns the number of items in the array. */
+ ZENCORE_API uint64_t Num() const;
+
+ /** Create an iterator for the fields of this array. */
+ ZENCORE_API CbFieldViewIterator CreateViewIterator() const;
+
+ /** Visit the fields of this array. */
+ ZENCORE_API void VisitFields(ICbVisitor& Visitor);
+
+ /** Access the array as an array field. */
+ inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); }
+
+ /** Construct an array from an array field. No type check is performed! */
+ static inline CbArrayView FromFieldView(const CbFieldView& Field) { return CbArrayView(Field); }
+
+ /** Whether the array has any fields. */
+ inline explicit operator bool() const { return Num() > 0; }
+
+ /** Returns the size of the array in bytes if serialized by itself with no name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the array if serialized by itself with no name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& Stream) const;
+
+ /**
+ * Whether this array is identical to the other array.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be done with the All mode to check these assumptions about the format of the inputs.
+ */
+ ZENCORE_API bool Equals(const CbArrayView& Other) const;
+
+ /** Copy the array into a buffer of exactly GetSize() bytes, with no name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the array into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ ///** Invoke the visitor for every attachment in the array. */
+ inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const
+ {
+ CreateViewIterator().IterateRangeAttachments(Visitor);
+ }
+
+ /** Returns a view of the array, including the type and name when present. */
+ using CbFieldView::GetView;
+
+ StringBuilderBase& ToJson(StringBuilderBase& Builder) const
+ {
+ CompactBinaryToJson(*this, Builder);
+ return Builder;
+ }
+
+private:
+ friend inline CbFieldViewIterator begin(const CbArrayView& Array) { return Array.CreateViewIterator(); }
+ friend inline CbFieldViewIterator end(const CbArrayView&) { return CbFieldViewIterator(); }
+
+ /** Construct an array from an array field. No type check is performed! Use via FromField. */
+ inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {}
+};
+
+/**
+ * Serialize a compact binary object to JSON.
+ */
+ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder);
+
+class CbObjectView : protected CbFieldView
+{
+ friend class CbFieldView;
+
+public:
+ /** @see CbField::CbField */
+ using CbFieldView::CbFieldView;
+
+ using CbFieldView::TryGetSerializedView;
+
+ /** Construct an object with no fields. */
+ ZENCORE_API CbObjectView();
+
+ /** Create an iterator for the fields of this object. */
+ ZENCORE_API CbFieldViewIterator CreateViewIterator() const;
+
+ /** Visit the fields of this object. */
+ ZENCORE_API void VisitFields(ICbVisitor& Visitor);
+
+ /**
+ * Find a field by case-sensitive name comparison.
+ *
+ * The cost of this operation scales linearly with the number of fields in the object. Prefer
+ * to iterate over the fields only once when consuming an object.
+ *
+ * @param Name The name of the field.
+ * @return The matching field if found, otherwise a field with no value.
+ */
+ ZENCORE_API CbFieldView FindView(std::string_view Name) const;
+
+ /** Find a field by case-insensitive name comparison. */
+ ZENCORE_API CbFieldView FindViewIgnoreCase(std::string_view Name) const;
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbFieldView operator[](std::string_view Name) const { return FindView(Name); }
+
+ /** Access the object as an object field. */
+ inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); }
+
+ /** Construct an object from an object field. No type check is performed! */
+ static inline CbObjectView FromFieldView(const CbFieldView& Field) { return CbObjectView(Field); }
+
+ /** Whether the object has any fields. */
+ ZENCORE_API explicit operator bool() const;
+
+ /** Returns the size of the object in bytes if serialized by itself with no name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the object if serialized by itself with no name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& HashStream) const;
+
+ /**
+ * Whether this object is identical to the other object.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be done with the All mode to check these assumptions about the format of the inputs.
+ */
+ ZENCORE_API bool Equals(const CbObjectView& Other) const;
+
+ /** Copy the object into a buffer of exactly GetSize() bytes, with no name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the field into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ ///** Invoke the visitor for every attachment in the object. */
+ inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const
+ {
+ CreateViewIterator().IterateRangeAttachments(Visitor);
+ }
+
+ /** Returns a view of the object, including the type and name when present. */
+ using CbFieldView::GetView;
+
+ /** Whether the field has a value. */
+ using CbFieldView::operator bool;
+
+ StringBuilderBase& ToJson(StringBuilderBase& Builder) const
+ {
+ CompactBinaryToJson(*this, Builder);
+ return Builder;
+ }
+
+private:
+ friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); }
+ friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); }
+
+ /** Construct an object from an object field. No type check is performed! Use via FromField. */
+ inline explicit CbObjectView(const CbFieldView& Field) : CbFieldView(Field) {}
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+/** A reference to a function that is used to allocate buffers for compact binary data. */
+using BufferAllocator = std::function<UniqueBuffer(uint64_t Size)>;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** A wrapper that holds a reference to the buffer that contains its compact binary value. */
+template<typename BaseType>
+class CbBuffer : public BaseType
+{
+public:
+ /** Construct a default value. */
+ CbBuffer() = default;
+
+ /**
+ * Construct a value from a pointer to its data and an optional externally-provided type.
+ *
+ * @param ValueBuffer A buffer that exactly contains the value.
+ * @param Type HasFieldType means that ValueBuffer contains the type. Otherwise, use the given type.
+ */
+ inline explicit CbBuffer(SharedBuffer ValueBuffer, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ if (ValueBuffer)
+ {
+ BaseType::operator=(BaseType(ValueBuffer.GetData(), Type));
+ ZEN_ASSERT(ValueBuffer.GetView().Contains(BaseType::GetView()));
+ Buffer = std::move(ValueBuffer);
+ }
+ }
+
+ /** Construct a value that holds a reference to the buffer that contains it. */
+ inline CbBuffer(const BaseType& Value, SharedBuffer OuterBuffer) : BaseType(Value)
+ {
+ if (OuterBuffer)
+ {
+ ZEN_ASSERT(OuterBuffer.GetView().Contains(BaseType::GetView()));
+ Buffer = std::move(OuterBuffer);
+ }
+ }
+
+ /** Construct a value that holds a reference to the buffer of the outer that contains it. */
+ template<typename OtherBaseType>
+ inline CbBuffer(const BaseType& Value, CbBuffer<OtherBaseType> OuterRef) : CbBuffer(Value, std::move(OuterRef.Buffer))
+ {
+ }
+
+ /** Reset this to a default value and null buffer. */
+ inline void Reset() { *this = CbBuffer(); }
+
+ /** Whether this reference has ownership of the memory in its buffer. */
+ inline bool IsOwned() const { return Buffer && Buffer.IsOwned(); }
+
+ /** Clone the value, if necessary, to a buffer that this reference has ownership of. */
+ inline void MakeOwned()
+ {
+ if (!IsOwned())
+ {
+ UniqueBuffer MutableBuffer = UniqueBuffer::Alloc(BaseType::GetSize());
+ BaseType::CopyTo(MutableBuffer);
+ BaseType::operator=(BaseType(MutableBuffer.GetData()));
+ Buffer = std::move(MutableBuffer);
+ }
+ }
+
+ /** Returns a buffer that exactly contains this value. */
+ inline SharedBuffer GetBuffer() const
+ {
+ const MemoryView View = BaseType::GetView();
+ const SharedBuffer& OuterBuffer = GetOuterBuffer();
+ return View == OuterBuffer.GetView() ? OuterBuffer : SharedBuffer::MakeView(View, OuterBuffer);
+ }
+
+ /** Returns the outer buffer (if any) that contains this value. */
+ inline const SharedBuffer& GetOuterBuffer() const& { return Buffer; }
+ inline SharedBuffer GetOuterBuffer() && { return std::move(Buffer); }
+
+private:
+ template<typename OtherType>
+ friend class CbBuffer;
+
+ SharedBuffer Buffer;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Factory functions for types derived from CbBuffer.
+ *
+ * This uses the curiously recurring template pattern to construct the correct type of reference.
+ * The derived type inherits from CbBufferRef and this type to expose the factory functions.
+ */
+template<typename RefType, typename BaseType>
+class CbBufferFactory
+{
+public:
+ /** Construct a value from an owned clone of its memory. */
+ static inline RefType Clone(const void* const Data) { return Clone(BaseType(Data)); }
+
+ /** Construct a value from an owned clone of its memory. */
+ static inline RefType Clone(const BaseType& Value)
+ {
+ RefType Ref = MakeView(Value);
+ Ref.MakeOwned();
+ return Ref;
+ }
+
+ /** Construct a value from a read-only view of its memory and its optional outer buffer. */
+ static inline RefType MakeView(const void* const Data, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return MakeView(BaseType(Data), std::move(OuterBuffer));
+ }
+
+ /** Construct a value from a read-only view of its memory and its optional outer buffer. */
+ static inline RefType MakeView(const BaseType& Value, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return RefType(Value, std::move(OuterBuffer));
+ }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class CbArray;
+class CbObject;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A field that can hold a reference to the memory that contains it.
+ *
+ * @see CbBufferRef
+ */
+class CbField : public CbBuffer<CbFieldView>, public CbBufferFactory<CbField, CbFieldView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Access the field as an object. Defaults to an empty object on error. */
+ inline CbObject AsObject() &;
+ inline CbObject AsObject() &&;
+
+ /** Access the field as an array. Defaults to an empty array on error. */
+ inline CbArray AsArray() &;
+ inline CbArray AsArray() &&;
+
+ /** Access the field as binary. Returns the provided default on error. */
+ inline SharedBuffer AsBinary(const SharedBuffer& Default = SharedBuffer()) &;
+ inline SharedBuffer AsBinary(const SharedBuffer& Default = SharedBuffer()) &&;
+};
+
+/**
+ * Iterator for CbFieldRef.
+ *
+ * @see CbFieldIterator
+ */
+class CbFieldIterator : public TCbFieldIterator<CbField>
+{
+public:
+ /** Construct a field range from an owned clone of a range. */
+ ZENCORE_API static CbFieldIterator CloneRange(const CbFieldViewIterator& It);
+
+ /** Construct a field range from an owned clone of a range. */
+ static inline CbFieldIterator CloneRange(const CbFieldIterator& It) { return CloneRange(CbFieldViewIterator(It)); }
+
+ /** Construct a field range that contains exactly one field. */
+ static inline CbFieldIterator MakeSingle(CbField Field) { return CbFieldIterator(std::move(Field)); }
+
+ /**
+ * Construct a field range from a buffer containing zero or more valid fields.
+ *
+ * @param Buffer A buffer containing zero or more valid fields.
+ * @param Type HasFieldType means that Buffer contains the type. Otherwise, use the given type.
+ */
+ static inline CbFieldIterator MakeRange(SharedBuffer Buffer, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ if (Buffer.GetSize())
+ {
+ const void* const DataEnd = Buffer.GetView().GetDataEnd();
+ return CbFieldIterator(CbField(std::move(Buffer), Type), DataEnd);
+ }
+ return CbFieldIterator();
+ }
+
+ /** Construct a field range from an iterator and its optional outer buffer. */
+ static inline CbFieldIterator MakeRangeView(const CbFieldViewIterator& It, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return CbFieldIterator(CbField(It, std::move(OuterBuffer)), GetFieldsEnd(It));
+ }
+
+ /** Construct an empty field range. */
+ constexpr CbFieldIterator() = default;
+
+ /** Clone the range, if necessary, to a buffer that this reference has ownership of. */
+ inline void MakeRangeOwned()
+ {
+ if (!IsOwned())
+ {
+ *this = CloneRange(*this);
+ }
+ }
+
+ /** Returns a buffer that exactly contains the field range. */
+ ZENCORE_API SharedBuffer GetRangeBuffer() const;
+
+private:
+ using TCbFieldIterator::TCbFieldIterator;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An array that can hold a reference to the memory that contains it.
+ *
+ * @see CbBuffer
+ */
+class CbArray : public CbBuffer<CbArrayView>, public CbBufferFactory<CbArray, CbArrayView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Create an iterator for the fields of this array. */
+ inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); }
+
+ /** Access the array as an array field. */
+ inline CbField AsField() const& { return CbField(CbArrayView::AsFieldView(), *this); }
+
+ /** Access the array as an array field. */
+ inline CbField AsField() && { return CbField(CbArrayView::AsFieldView(), std::move(*this)); }
+
+private:
+ friend inline CbFieldIterator begin(const CbArray& Array) { return Array.CreateIterator(); }
+ friend inline CbFieldIterator end(const CbArray&) { return CbFieldIterator(); }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An object that can hold a reference to the memory that contains it.
+ *
+ * @see CbBuffer
+ */
+class CbObject : public CbBuffer<CbObjectView>, public CbBufferFactory<CbObject, CbObjectView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Create an iterator for the fields of this object. */
+ inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); }
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbField Find(std::string_view Name) const
+ {
+ if (CbFieldView Field = FindView(Name))
+ {
+ return CbField(Field, *this);
+ }
+ return CbField();
+ }
+
+ /** Find a field by case-insensitive name comparison. */
+ inline CbField FindIgnoreCase(std::string_view Name) const
+ {
+ if (CbFieldView Field = FindViewIgnoreCase(Name))
+ {
+ return CbField(Field, *this);
+ }
+ return CbField();
+ }
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbFieldView operator[](std::string_view Name) const { return Find(Name); }
+
+ /** Access the object as an object field. */
+ inline CbField AsField() const& { return CbField(CbObjectView::AsFieldView(), *this); }
+
+ /** Access the object as an object field. */
+ inline CbField AsField() && { return CbField(CbObjectView::AsFieldView(), std::move(*this)); }
+
+private:
+ friend inline CbFieldIterator begin(const CbObject& Object) { return Object.CreateIterator(); }
+ friend inline CbFieldIterator end(const CbObject&) { return CbFieldIterator(); }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+inline CbObject
+CbField::AsObject() &
+{
+ return IsObject() ? CbObject(AsObjectView(), *this) : CbObject();
+}
+
+inline CbObject
+CbField::AsObject() &&
+{
+ return IsObject() ? CbObject(AsObjectView(), std::move(*this)) : CbObject();
+}
+
+inline CbArray
+CbField::AsArray() &
+{
+ return IsArray() ? CbArray(AsArrayView(), *this) : CbArray();
+}
+
+inline CbArray
+CbField::AsArray() &&
+{
+ return IsArray() ? CbArray(AsArrayView(), std::move(*this)) : CbArray();
+}
+
+inline SharedBuffer
+CbField::AsBinary(const SharedBuffer& Default) &
+{
+ const MemoryView View = AsBinaryView();
+ return !HasError() ? SharedBuffer::MakeView(View, GetOuterBuffer()) : Default;
+}
+
+inline SharedBuffer
+CbField::AsBinary(const SharedBuffer& Default) &&
+{
+ const MemoryView View = AsBinaryView();
+ return !HasError() ? SharedBuffer::MakeView(View, std::move(*this).GetOuterBuffer()) : Default;
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Load a compact binary field from an archive.
+ *
+ * The field may be an array or an object, which the caller can convert to by using AsArray or
+ * AsObject as appropriate. The buffer allocator is called to provide the buffer for the field
+ * to load into once its size has been determined.
+ *
+ * @param Ar Archive to read the field from. An error state is set on failure.
+ * @param Allocator Allocator for the buffer that the field is loaded into.
+ * @return A field with a reference to the allocated buffer, or a default field on failure.
+ */
+ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator);
+
+ZENCORE_API CbObject LoadCompactBinaryObject(IoBuffer&& Payload);
+ZENCORE_API CbObject LoadCompactBinaryObject(const IoBuffer& Payload);
+ZENCORE_API CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload);
+ZENCORE_API CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload);
+
+/**
+ * Load a compact binary from JSON.
+ */
+ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json, std::string& Error);
+ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Determine the size in bytes of the compact binary field at the start of the view.
+ *
+ * This may be called on an incomplete or invalid field, in which case the returned size is zero.
+ * A size can always be extracted from a valid field with no name if a view of at least the first
+ * 10 bytes is provided, regardless of field size. For fields with names, the size of view needed
+ * to calculate a size is at most 10 + MaxNameLen + MeasureVarUInt(MaxNameLen).
+ *
+ * This function can be used when streaming a field, for example, to determine the size of buffer
+ * to fill before attempting to construct a field from it.
+ *
+ * @param View A memory view that may contain the start of a field.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ */
+ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType);
+
+/**
+ * Try to determine the type and size of the compact binary field at the start of the view.
+ *
+ * This may be called on an incomplete or invalid field, in which case it will return false, with
+ * OutSize being 0 for invalid fields, otherwise the minimum view size necessary to make progress
+ * in measuring the field on the next call to this function.
+ *
+ * @note A return of true from this function does not indicate that the entire field is valid.
+ *
+ * @param InView A memory view that may contain the start of a field.
+ * @param OutType The type (with flags) of the field. None is written until a value is available.
+ * @param OutSize The total field size for a return of true, 0 for invalid fields, or the size to
+ * make progress in measuring the field on the next call to this function.
+ * @param InType HasFieldType means that InView contains the type. Otherwise, use the given type.
+ * @return true if the size of the field was determined, otherwise false.
+ */
+ZENCORE_API bool TryMeasureCompactBinary(MemoryView InView,
+ CbFieldType& OutType,
+ uint64_t& OutSize,
+ CbFieldType InType = CbFieldType::HasFieldType);
+
+inline CbFieldViewIterator
+begin(CbFieldView& View)
+{
+ if (View.IsArray())
+ {
+ return View.AsArrayView().CreateViewIterator();
+ }
+ else if (View.IsObject())
+ {
+ return View.AsObjectView().CreateViewIterator();
+ }
+
+ return CbFieldViewIterator();
+}
+
+inline CbFieldViewIterator
+end(CbFieldView&)
+{
+ return CbFieldViewIterator();
+}
+
+void uson_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h
new file mode 100644
index 000000000..4be8c2ba5
--- /dev/null
+++ b/src/zencore/include/zencore/compactbinarybuilder.h
@@ -0,0 +1,661 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+
+#include <zencore/enumflags.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/refcount.h>
+#include <zencore/sha1.h>
+
+#include <atomic>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class CbAttachment;
+class BinaryWriter;
+
+/**
+ * A writer for compact binary object, arrays, and fields.
+ *
+ * The writer produces a sequence of fields that can be saved to a provided memory buffer or into
+ * a new owned buffer. The typical use case is to write a single object, which can be accessed by
+ * calling Save().AsObjectRef() or Save(Buffer).AsObject().
+ *
+ * The writer will assert on most incorrect usage and will always produce valid compact binary if
+ * provided with valid input. The writer does not check for invalid UTF-8 string encoding, object
+ * fields with duplicate names, or invalid compact binary being copied from another source.
+ *
+ * It is most convenient to use the streaming API for the writer, as demonstrated in the example.
+ *
+ * When writing a small amount of compact binary data, TCbWriter can be more efficient as it uses
+ * a fixed-size stack buffer for storage before spilling onto the heap.
+ *
+ * @see TCbWriter
+ *
+ * Example:
+ *
+ * CbObjectRef WriteObject()
+ * {
+ * CbWriter<256> Writer;
+ * Writer.BeginObject();
+ *
+ * Writer << "Resize" << true;
+ * Writer << "MaxWidth" << 1024;
+ * Writer << "MaxHeight" << 1024;
+ *
+ * Writer.BeginArray();
+ * Writer << "FormatA" << "FormatB" << "FormatC";
+ * Writer.EndArray();
+ *
+ * Writer.EndObject();
+ * return Writer.Save().AsObjectRef();
+ * }
+ */
+class CbWriter
+{
+public:
+ ZENCORE_API CbWriter();
+ ZENCORE_API ~CbWriter();
+
+ CbWriter(const CbWriter&) = delete;
+ CbWriter& operator=(const CbWriter&) = delete;
+
+ /** Empty the writer without releasing any allocated memory. */
+ ZENCORE_API void Reset();
+
+ /**
+ * Serialize the field(s) to an owned buffer and return it as an iterator.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ * The writer remains valid for further use when this function returns.
+ */
+ ZENCORE_API CbFieldIterator Save();
+
+ /**
+ * Serialize the field(s) to memory.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ * The writer remains valid for further use when this function returns.
+ *
+ * @param Buffer A mutable memory view to write to. Must be exactly GetSaveSize() bytes.
+ * @return An iterator for the field(s) written to the buffer.
+ */
+ ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer);
+
+ ZENCORE_API void Save(BinaryWriter& Writer);
+
+ /**
+ * The size of buffer (in bytes) required to serialize the fields that have been written.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ */
+ ZENCORE_API uint64_t GetSaveSize() const;
+
+ /**
+ * Sets the name of the next field to be written.
+ *
+ * It is not valid to call this function when writing a field inside an array.
+ * Names must be valid UTF-8 and must be unique within an object.
+ */
+ ZENCORE_API CbWriter& SetName(std::string_view Name);
+
+ /** Copy the value (not the name) of an existing field. */
+ inline void AddField(std::string_view Name, const CbFieldView& Value)
+ {
+ SetName(Name);
+ AddField(Value);
+ }
+
+ ZENCORE_API void AddField(const CbFieldView& Value);
+
+ /** Copy the value (not the name) of an existing field. Holds a reference if owned. */
+ inline void AddField(std::string_view Name, const CbField& Value)
+ {
+ SetName(Name);
+ AddField(Value);
+ }
+ ZENCORE_API void AddField(const CbField& Value);
+
+ /** Begin a new object. Must have a matching call to EndObject. */
+ inline void BeginObject(std::string_view Name)
+ {
+ SetName(Name);
+ BeginObject();
+ }
+ ZENCORE_API void BeginObject();
+ /** End an object after its fields have been written. */
+ ZENCORE_API void EndObject();
+
+ /** Copy the value (not the name) of an existing object. */
+ inline void AddObject(std::string_view Name, const CbObjectView& Value)
+ {
+ SetName(Name);
+ AddObject(Value);
+ }
+ ZENCORE_API void AddObject(const CbObjectView& Value);
+ /** Copy the value (not the name) of an existing object. Holds a reference if owned. */
+ inline void AddObject(std::string_view Name, const CbObject& Value)
+ {
+ SetName(Name);
+ AddObject(Value);
+ }
+ ZENCORE_API void AddObject(const CbObject& Value);
+
+ /** Begin a new array. Must have a matching call to EndArray. */
+ inline void BeginArray(std::string_view Name)
+ {
+ SetName(Name);
+ BeginArray();
+ }
+ ZENCORE_API void BeginArray();
+ /** End an array after its fields have been written. */
+ ZENCORE_API void EndArray();
+
+ /** Copy the value (not the name) of an existing array. */
+ inline void AddArray(std::string_view Name, const CbArrayView& Value)
+ {
+ SetName(Name);
+ AddArray(Value);
+ }
+ ZENCORE_API void AddArray(const CbArrayView& Value);
+ /** Copy the value (not the name) of an existing array. Holds a reference if owned. */
+ inline void AddArray(std::string_view Name, const CbArray& Value)
+ {
+ SetName(Name);
+ AddArray(Value);
+ }
+ ZENCORE_API void AddArray(const CbArray& Value);
+
+ /** Write a null field. */
+ inline void AddNull(std::string_view Name)
+ {
+ SetName(Name);
+ AddNull();
+ }
+ ZENCORE_API void AddNull();
+
+ /** Write a binary field by copying Size bytes from Value. */
+ inline void AddBinary(std::string_view Name, const void* Value, uint64_t Size)
+ {
+ SetName(Name);
+ AddBinary(Value, Size);
+ }
+ ZENCORE_API void AddBinary(const void* Value, uint64_t Size);
+ /** Write a binary field by copying the view. */
+ inline void AddBinary(std::string_view Name, MemoryView Value)
+ {
+ SetName(Name);
+ AddBinary(Value);
+ }
+ inline void AddBinary(MemoryView Value) { AddBinary(Value.GetData(), Value.GetSize()); }
+
+ /** Write a binary field by copying the buffer. Holds a reference if owned. */
+ inline void AddBinary(std::string_view Name, IoBuffer Value)
+ {
+ SetName(Name);
+ AddBinary(std::move(Value));
+ }
+ ZENCORE_API void AddBinary(IoBuffer Value);
+ ZENCORE_API void AddBinary(SharedBuffer Value);
+
+ inline void AddBinary(std::string_view Name, const CompositeBuffer& Buffer)
+ {
+ SetName(Name);
+ AddBinary(Buffer);
+ }
+ ZENCORE_API void AddBinary(const CompositeBuffer& Buffer);
+
+ /** Write a string field by copying the UTF-8 value. */
+ inline void AddString(std::string_view Name, std::string_view Value)
+ {
+ SetName(Name);
+ AddString(Value);
+ }
+ ZENCORE_API void AddString(std::string_view Value);
+ /** Write a string field by converting the UTF-16 value to UTF-8. */
+ inline void AddString(std::string_view Name, std::wstring_view Value)
+ {
+ SetName(Name);
+ AddString(Value);
+ }
+ ZENCORE_API void AddString(std::wstring_view Value);
+
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, int32_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(int32_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, int64_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(int64_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, uint32_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(uint32_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, uint64_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(uint64_t Value);
+
+ /** Write a float field from a 32-bit float value. */
+ inline void AddFloat(std::string_view Name, float Value)
+ {
+ SetName(Name);
+ AddFloat(Value);
+ }
+ ZENCORE_API void AddFloat(float Value);
+
+ /** Write a float field from a 64-bit float value. */
+ inline void AddFloat(std::string_view Name, double Value)
+ {
+ SetName(Name);
+ AddFloat(Value);
+ }
+ ZENCORE_API void AddFloat(double Value);
+
+ /** Write a bool field. */
+ inline void AddBool(std::string_view Name, bool bValue)
+ {
+ SetName(Name);
+ AddBool(bValue);
+ }
+ ZENCORE_API void AddBool(bool bValue);
+
+ /** Write a field referencing a compact binary attachment by its hash. */
+ inline void AddObjectAttachment(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddObjectAttachment(Value);
+ }
+ ZENCORE_API void AddObjectAttachment(const IoHash& Value);
+
+ /** Write a field referencing a binary attachment by its hash. */
+ inline void AddBinaryAttachment(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddBinaryAttachment(Value);
+ }
+ ZENCORE_API void AddBinaryAttachment(const IoHash& Value);
+
+ /** Write a field referencing the attachment by its hash. */
+ inline void AddAttachment(std::string_view Name, const CbAttachment& Attachment)
+ {
+ SetName(Name);
+ AddAttachment(Attachment);
+ }
+ ZENCORE_API void AddAttachment(const CbAttachment& Attachment);
+
+ /** Write a hash field. */
+ inline void AddHash(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddHash(Value);
+ }
+ ZENCORE_API void AddHash(const IoHash& Value);
+
+ /** Write a UUID field. */
+ inline void AddUuid(std::string_view Name, const Guid& Value)
+ {
+ SetName(Name);
+ AddUuid(Value);
+ }
+ ZENCORE_API void AddUuid(const Guid& Value);
+
+ /** Write an ObjectId field. */
+ inline void AddObjectId(std::string_view Name, const Oid& Value)
+ {
+ SetName(Name);
+ AddObjectId(Value);
+ }
+ ZENCORE_API void AddObjectId(const Oid& Value);
+
+ /** Write a date/time field with the specified count of 100ns ticks since the epoch. */
+ inline void AddDateTimeTicks(std::string_view Name, int64_t Ticks)
+ {
+ SetName(Name);
+ AddDateTimeTicks(Ticks);
+ }
+ ZENCORE_API void AddDateTimeTicks(int64_t Ticks);
+
+ /** Write a date/time field. */
+ inline void AddDateTime(std::string_view Name, DateTime Value)
+ {
+ SetName(Name);
+ AddDateTime(Value);
+ }
+ ZENCORE_API void AddDateTime(DateTime Value);
+
+ /** Write a time span field with the specified count of 100ns ticks. */
+ inline void AddTimeSpanTicks(std::string_view Name, int64_t Ticks)
+ {
+ SetName(Name);
+ AddTimeSpanTicks(Ticks);
+ }
+ ZENCORE_API void AddTimeSpanTicks(int64_t Ticks);
+
+ /** Write a time span field. */
+ inline void AddTimeSpan(std::string_view Name, TimeSpan Value)
+ {
+ SetName(Name);
+ AddTimeSpan(Value);
+ }
+ ZENCORE_API void AddTimeSpan(TimeSpan Value);
+
+ /** Private flags that are public to work with ENUM_CLASS_FLAGS. */
+ enum class StateFlags : uint8_t;
+
+protected:
+ /** Reserve the specified size up front until the format is optimized. */
+ ZENCORE_API explicit CbWriter(int64_t InitialSize);
+
+private:
+ friend CbWriter& operator<<(CbWriter& Writer, std::string_view NameOrValue);
+
+ /** Begin writing a field. May be called twice for named fields. */
+ void BeginField();
+
+ /** Finish writing a field by writing its type. */
+ void EndField(CbFieldType Type);
+
+ /** Set the field name if valid in this state, otherwise write add a string field. */
+ ZENCORE_API void SetNameOrAddString(std::string_view NameOrValue);
+
+ /** Returns a view of the name of the active field, if any, otherwise the empty view. */
+ std::string_view GetActiveName() const;
+
+ /** Remove field types after the first to make the sequence uniform. */
+ void MakeFieldsUniform(int64_t FieldBeginOffset, int64_t FieldEndOffset);
+
+ /** State of the object, array, or top-level field being written. */
+ struct WriterState
+ {
+ StateFlags Flags{};
+ /** The type of the fields in the sequence if uniform, otherwise None. */
+ CbFieldType UniformType{};
+ /** The offset of the start of the current field. */
+ int64_t Offset{};
+ /** The number of fields written in this state. */
+ uint64_t Count{};
+ };
+
+private:
+ // This is a prototype-quality format for the writer. Using an array of bytes is inefficient,
+ // and will lead to many unnecessary copies and moves of the data to resize the array, insert
+ // object and array sizes, and remove field types for uniform objects and uniform arrays. The
+ // optimized format will be a list of power-of-two blocks and an optional first block that is
+ // provided externally, such as on the stack. That format will store the offsets that require
+ // object or array sizes to be inserted and field types to be removed, and will perform those
+ // operations only when saving to a buffer.
+ std::vector<uint8_t> Data;
+ std::vector<WriterState> States;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A writer for compact binary object, arrays, and fields that uses a fixed-size stack buffer.
+ *
+ * @see CbWriter
+ */
+template<uint32_t InlineBufferSize>
+class FixedCbWriter : public CbWriter
+{
+public:
+ inline FixedCbWriter() : CbWriter(InlineBufferSize) {}
+
+ FixedCbWriter(const FixedCbWriter&) = delete;
+ FixedCbWriter& operator=(const FixedCbWriter&) = delete;
+
+private:
+ // Reserve the inline buffer now even though we are unable to use it. This will avoid causing
+ // new stack overflows when this functionality is properly implemented in the future.
+ uint8_t Buffer[InlineBufferSize];
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class CbObjectWriter : public CbWriter
+{
+public:
+ CbObjectWriter() { BeginObject(); }
+
+ ZENCORE_API CbObject Save()
+ {
+ Finalize();
+ return CbWriter::Save().AsObject();
+ }
+
+ ZENCORE_API void Save(BinaryWriter& Writer)
+ {
+ Finalize();
+ return CbWriter::Save(Writer);
+ }
+
+ ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer)
+ {
+ ZEN_ASSERT(m_Finalized);
+ return CbWriter::Save(Buffer);
+ }
+
+ uint64_t GetSaveSize()
+ {
+ ZEN_ASSERT(m_Finalized);
+ return CbWriter::GetSaveSize();
+ }
+
+ void Finalize()
+ {
+ if (m_Finalized == false)
+ {
+ EndObject();
+ m_Finalized = true;
+ }
+ }
+
+ CbObjectWriter(const CbWriter&) = delete;
+ CbObjectWriter& operator=(const CbWriter&) = delete;
+
+private:
+ bool m_Finalized = false;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Write the field name if valid in this state, otherwise write the string value. */
+inline CbWriter&
+operator<<(CbWriter& Writer, std::string_view NameOrValue)
+{
+ Writer.SetNameOrAddString(NameOrValue);
+ return Writer;
+}
+
+/** Write the field name if valid in this state, otherwise write the string value. */
+inline CbWriter&
+operator<<(CbWriter& Writer, const char* NameOrValue)
+{
+ return Writer << std::string_view(NameOrValue);
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbFieldView& Value)
+{
+ Writer.AddField(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbField& Value)
+{
+ Writer.AddField(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbObjectView& Value)
+{
+ Writer.AddObject(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbObject& Value)
+{
+ Writer.AddObject(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbArrayView& Value)
+{
+ Writer.AddArray(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbArray& Value)
+{
+ Writer.AddArray(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, std::nullptr_t)
+{
+ Writer.AddNull();
+ return Writer;
+}
+
+#if defined(__clang__) && defined(__APPLE__)
+/* Apple Clang has different types for uint64_t and size_t so an override is
+ needed here. Without it, Clang can't disambiguate integer overloads */
+inline CbWriter&
+operator<<(CbWriter& Writer, std::size_t Value)
+{
+ Writer.AddInteger(uint64_t(Value));
+ return Writer;
+}
+#endif
+
+inline CbWriter&
+operator<<(CbWriter& Writer, std::wstring_view Value)
+{
+ Writer.AddString(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const wchar_t* Value)
+{
+ Writer.AddString(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, int32_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, int64_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, uint32_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, uint64_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, float Value)
+{
+ Writer.AddFloat(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, double Value)
+{
+ Writer.AddFloat(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, bool Value)
+{
+ Writer.AddBool(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbAttachment& Attachment)
+{
+ Writer.AddAttachment(Attachment);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const IoHash& Value)
+{
+ Writer.AddHash(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const Guid& Value)
+{
+ Writer.AddUuid(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const Oid& Value)
+{
+ Writer.AddObjectId(Value);
+ return Writer;
+}
+
+ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value);
+ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void usonbuilder_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h
new file mode 100644
index 000000000..16f723edc
--- /dev/null
+++ b/src/zencore/include/zencore/compactbinarypackage.h
@@ -0,0 +1,341 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/compress.h>
+#include <zencore/iohash.h>
+
+#include <functional>
+#include <span>
+#include <variant>
+
+#ifdef GetObject
+# error "windows.h pollution"
+# undef GetObject
+#endif
+
+namespace zen {
+
+class CbWriter;
+class BinaryReader;
+class BinaryWriter;
+class IoBuffer;
+class CbAttachment;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An attachment is either binary or compact binary and is identified by its hash.
+ *
+ * A compact binary attachment is also a valid binary attachment and may be accessed as binary.
+ *
+ * Attachments are serialized as one or two compact binary fields with no name. A Binary field is
+ * written first with its content. The content hash is omitted when the content size is zero, and
+ * is otherwise written as a BinaryReference or CompactBinaryReference depending on the type.
+ */
+class CbAttachment
+{
+public:
+ /** Construct a null attachment. */
+ CbAttachment() = default;
+
+ /** Construct a compact binary attachment. Value is cloned if not owned. */
+ inline explicit CbAttachment(const CbObject& InValue) : CbAttachment(InValue, nullptr) {}
+
+ /** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */
+ inline explicit CbAttachment(const CbObject& InValue, const IoHash& Hash) : CbAttachment(InValue, &Hash) {}
+
+ /** Construct a raw binary attachment. Value is cloned if not owned. */
+ ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue);
+
+ /** Construct a raw binary attachment. Value is cloned if not owned. Hash must match Value. */
+ ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue, const IoHash& Hash);
+
+ /** Construct a raw binary attachment. Value is cloned if not owned. */
+ ZENCORE_API explicit CbAttachment(const CompositeBuffer& InValue);
+
+ /** Construct a raw binary attachment. Value is cloned if not owned. */
+ ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue);
+
+ /** Construct a raw binary attachment. Value is cloned if not owned. */
+ ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash);
+
+ /** Construct a compressed binary attachment. Value is cloned if not owned. */
+ ZENCORE_API explicit CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash);
+ ZENCORE_API explicit CbAttachment(CompressedBuffer&& InValue, const IoHash& Hash);
+
+ /** Reset this to a null attachment. */
+ inline void Reset() { *this = CbAttachment(); }
+
+ /** Whether the attachment has a value. */
+ inline explicit operator bool() const { return !IsNull(); }
+
+ /** Whether the attachment has a value. */
+ ZENCORE_API [[nodiscard]] bool IsNull() const;
+
+ /** Access the attachment as binary. Defaults to a null buffer on error. */
+ ZENCORE_API [[nodiscard]] SharedBuffer AsBinary() const;
+
+ /** Access the attachment as raw binary. Defaults to a null buffer on error. */
+ ZENCORE_API [[nodiscard]] CompositeBuffer AsCompositeBinary() const;
+
+ /** Access the attachment as compressed binary. Defaults to a null buffer if the attachment is null. */
+ ZENCORE_API [[nodiscard]] CompressedBuffer AsCompressedBinary() const;
+
+ /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */
+ ZENCORE_API [[nodiscard]] CbObject AsObject() const;
+
+ /** Returns true if the attachment is binary */
+ ZENCORE_API [[nodiscard]] bool IsBinary() const;
+
+ /** Returns true if the attachment is compressed binary */
+ ZENCORE_API [[nodiscard]] bool IsCompressedBinary() const;
+
+ /** Returns whether the attachment is an object. */
+ ZENCORE_API [[nodiscard]] bool IsObject() const;
+
+ /** Returns the hash of the attachment value. */
+ ZENCORE_API [[nodiscard]] IoHash GetHash() const;
+
+ /** Compares attachments by their hash. Any discrepancy in type must be handled externally. */
+ inline bool operator==(const CbAttachment& Attachment) const { return GetHash() == Attachment.GetHash(); }
+ inline bool operator!=(const CbAttachment& Attachment) const { return GetHash() != Attachment.GetHash(); }
+ inline bool operator<(const CbAttachment& Attachment) const { return GetHash() < Attachment.GetHash(); }
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ *
+ * The attachment references the input iterator if it is owned, and otherwise clones the value.
+ *
+ * The iterator is advanced as attachment fields are consumed from it.
+ */
+ ZENCORE_API bool TryLoad(CbFieldIterator& Fields);
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ */
+ ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ */
+ ZENCORE_API bool TryLoad(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ /** Save the attachment into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(CbWriter& Writer) const;
+
+ /** Save the attachment into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(BinaryWriter& Writer) const;
+
+private:
+ ZENCORE_API CbAttachment(const CbObject& Value, const IoHash* Hash);
+
+ IoHash Hash;
+ std::variant<std::nullptr_t, CbObject, CompositeBuffer, CompressedBuffer> Value;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A package is a compact binary object with attachments for its external references.
+ *
+ * A package is basically a Merkle tree with compact binary as its root and other non-leaf nodes,
+ * and either binary or compact binary as its leaf nodes. A node references its child nodes using
+ * BinaryHash or FieldHash fields in its compact binary representation.
+ *
+ * It is invalid for a package to include attachments that are not referenced by its object or by
+ * one of its referenced compact binary attachments. When attachments are added explicitly, it is
+ * the responsibility of the package creator to follow this requirement. Attachments that are not
+ * referenced may not survive a round-trip through certain storage systems.
+ *
+ * It is valid for a package to exclude referenced attachments, but then it is the responsibility
+ * of the package consumer to have a mechanism for resolving those references when necessary.
+ *
+ * A package is serialized as a sequence of compact binary fields with no name. The object may be
+ * both preceded and followed by attachments. The object itself is written as an Object field and
+ * followed by its hash in a CompactBinaryReference field when the object is non-empty. A package
+ * ends with a Null field. The canonical order of components is the object and its hash, followed
+ * by the attachments ordered by hash, followed by a Null field. It is valid for the a package to
+ * have its components serialized in any order, provided there is at most one object and the null
+ * field is written last.
+ */
+class CbPackage
+{
+public:
+ /**
+ * A function that resolves a hash to a buffer containing the data matching that hash.
+ *
+ * The resolver may return a null buffer to skip resolving an attachment for the hash.
+ */
+ using AttachmentResolver = std::function<SharedBuffer(const IoHash& Hash)>;
+
+ /** Construct a null package. */
+ CbPackage() = default;
+
+ /**
+ * Construct a package from a root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ */
+ inline explicit CbPackage(CbObject InObject) { SetObject(std::move(InObject)); }
+
+ /**
+ * Construct a package from a root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline explicit CbPackage(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), InResolver); }
+
+ /**
+ * Construct a package from a root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ */
+ inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), InObjectHash); }
+
+ /**
+ * Construct a package from a root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver)
+ {
+ SetObject(std::move(InObject), InObjectHash, InResolver);
+ }
+
+ /** Reset this to a null package. */
+ inline void Reset() { *this = CbPackage(); }
+
+ /** Whether the package has a non-empty object or attachments. */
+ inline explicit operator bool() const { return !IsNull(); }
+
+ /** Whether the package has an empty object and no attachments. */
+ inline bool IsNull() const { return !Object && Attachments.size() == 0; }
+
+ /** Returns the compact binary object for the package. */
+ inline const CbObject& GetObject() const { return Object; }
+
+ /** Returns the has of the compact binary object for the package. */
+ inline const IoHash& GetObjectHash() const { return ObjectHash; }
+
+ /**
+ * Set the root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ */
+ inline void SetObject(CbObject InObject) { SetObject(std::move(InObject), nullptr, nullptr); }
+
+ /**
+ * Set the root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline void SetObject(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), nullptr, &InResolver); }
+
+ /**
+ * Set the root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ */
+ inline void SetObject(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), &InObjectHash, nullptr); }
+
+ /**
+ * Set the root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline void SetObject(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver)
+ {
+ SetObject(std::move(InObject), &InObjectHash, &InResolver);
+ }
+
+ /** Returns the attachments in this package. */
+ inline std::span<const CbAttachment> GetAttachments() const { return Attachments; }
+
+ /**
+ * Find an attachment by its hash.
+ *
+ * @return The attachment, or null if the attachment is not found.
+ * @note The returned pointer is only valid until the attachments on this package are modified.
+ */
+ ZENCORE_API const CbAttachment* FindAttachment(const IoHash& Hash) const;
+
+ /** Find an attachment if it exists in the package. */
+ inline const CbAttachment* FindAttachment(const CbAttachment& Attachment) const { return FindAttachment(Attachment.GetHash()); }
+
+ /** Add the attachment to this package. */
+ inline void AddAttachment(const CbAttachment& Attachment) { AddAttachment(Attachment, nullptr); }
+
+ /** Add the attachment to this package, along with any references that can be resolved. */
+ inline void AddAttachment(const CbAttachment& Attachment, AttachmentResolver Resolver) { AddAttachment(Attachment, &Resolver); }
+
+ void AddAttachments(std::span<const CbAttachment> Attachments);
+
+ /**
+ * Remove an attachment by hash.
+ *
+ * @return Number of attachments removed, which will be either 0 or 1.
+ */
+ ZENCORE_API int32_t RemoveAttachment(const IoHash& Hash);
+ inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); }
+
+ /** Compares packages by their object and attachment hashes. */
+ ZENCORE_API bool Equals(const CbPackage& Package) const;
+ inline bool operator==(const CbPackage& Package) const { return Equals(Package); }
+ inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); }
+
+ /**
+ * Load the object and attachments from compact binary as written by Save.
+ *
+ * The object and attachments reference the input iterator, if it is owned, and otherwise clones
+ * the object and attachments individually to make owned copies.
+ *
+ * The iterator is advanced as object and attachment fields are consumed from it.
+ */
+ ZENCORE_API bool TryLoad(CbFieldIterator& Fields);
+ ZENCORE_API bool TryLoad(IoBuffer Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr);
+ ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr);
+
+ /** Save the object and attachments into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(CbWriter& Writer) const;
+
+ /** Save the object and attachments into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(BinaryWriter& Writer) const;
+
+private:
+ ZENCORE_API void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver);
+ ZENCORE_API void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver);
+
+ void GatherAttachments(const CbObject& Object, AttachmentResolver Resolver);
+
+ /** Attachments ordered by their hash. */
+ std::vector<CbAttachment> Attachments;
+ CbObject Object;
+ IoHash ObjectHash;
+};
+
+namespace legacy {
+ void SaveCbAttachment(const CbAttachment& Attachment, CbWriter& Writer);
+ void SaveCbPackage(const CbPackage& Package, CbWriter& Writer);
+ void SaveCbPackage(const CbPackage& Package, BinaryWriter& Ar);
+ bool TryLoadCbPackage(CbPackage& Package, IoBuffer Buffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper = nullptr);
+ bool TryLoadCbPackage(CbPackage& Package,
+ BinaryReader& Reader,
+ BufferAllocator Allocator,
+ CbPackage::AttachmentResolver* Mapper = nullptr);
+} // namespace legacy
+
+void usonpackage_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/compactbinaryvalidation.h b/src/zencore/include/zencore/compactbinaryvalidation.h
new file mode 100644
index 000000000..b1fab9572
--- /dev/null
+++ b/src/zencore/include/zencore/compactbinaryvalidation.h
@@ -0,0 +1,197 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/enumflags.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/refcount.h>
+#include <zencore/sha1.h>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+/** Flags for validating compact binary data. */
+enum class CbValidateMode : uint32_t
+{
+ /** Skip validation if no other validation modes are enabled. */
+ None = 0,
+
+ /**
+ * Validate that the value can be read and stays inside the bounds of the memory view.
+ *
+ * This is the minimum level of validation required to be able to safely read a field, array,
+ * or object without the risk of crashing or reading out of bounds.
+ */
+ Default = 1 << 0,
+
+ /**
+ * Validate that object fields have unique non-empty names and array fields have no names.
+ *
+ * Name validation failures typically do not inhibit reading the input, but duplicated fields
+ * cannot be looked up by name other than the first, and converting to other data formats can
+ * fail in the presence of naming issues.
+ */
+ Names = 1 << 1,
+
+ /**
+ * Validate that fields are serialized in the canonical format.
+ *
+ * Format validation failures typically do not inhibit reading the input. Values that fail in
+ * this mode require more memory than in the canonical format, and comparisons of such values
+ * for equality are not reliable. Examples of failures include uniform arrays or objects that
+ * were not encoded uniformly, variable-length integers that could be encoded in fewer bytes,
+ * or 64-bit floats that could be encoded in 32 bits without loss of precision.
+ */
+ Format = 1 << 2,
+
+ /**
+ * Validate that there is no padding after the value before the end of the memory view.
+ *
+ * Padding validation failures have no impact on the ability to read the input, but are using
+ * more memory than necessary.
+ */
+ Padding = 1 << 3,
+
+ /**
+ * Validate that a package or attachment has the expected fields.
+ */
+ Package = 1 << 4,
+
+ /**
+ * Validate that a package or attachment matches its saved hashes.
+ */
+ PackageHash = 1 << 5,
+
+ /** Perform all validation described above. */
+ All = Default | Names | Format | Padding | Package | PackageHash,
+};
+
+ENUM_CLASS_FLAGS(CbValidateMode);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Flags for compact binary validation errors. Multiple flags may be combined. */
+enum class CbValidateError : uint32_t
+{
+ /** The input had no validation errors. */
+ None = 0,
+
+ // Mode: Default
+
+ /** The input cannot be read without reading out of bounds. */
+ OutOfBounds = 1 << 0,
+ /** The input has a field with an unrecognized or invalid type. */
+ InvalidType = 1 << 1,
+
+ // Mode: Names
+
+ /** An object had more than one field with the same name. */
+ DuplicateName = 1 << 2,
+ /** An object had a field with no name. */
+ MissingName = 1 << 3,
+ /** An array field had a name. */
+ ArrayName = 1 << 4,
+
+ // Mode: Format
+
+ /** A name or string payload is not valid UTF-8. */
+ InvalidString = 1 << 5,
+ /** A size or integer payload can be encoded in fewer bytes. */
+ InvalidInteger = 1 << 6,
+ /** A float64 payload can be encoded as a float32 without loss of precision. */
+ InvalidFloat = 1 << 7,
+ /** An object has the same type for every field but is not uniform. */
+ NonUniformObject = 1 << 8,
+ /** An array has the same type for every field and non-empty payloads but is not uniform. */
+ NonUniformArray = 1 << 9,
+
+ // Mode: Padding
+
+ /** A value did not use the entire memory view given for validation. */
+ Padding = 1 << 10,
+
+ // Mode: Package
+
+ /** The package or attachment had missing fields or fields out of order. */
+ InvalidPackageFormat = 1 << 11,
+ /** The object or an attachment did not match the hash stored for it. */
+ InvalidPackageHash = 1 << 12,
+ /** The package contained more than one copy of the same attachment. */
+ DuplicateAttachments = 1 << 13,
+ /** The package contained more than one object. */
+ MultiplePackageObjects = 1 << 14,
+ /** The package contained an object with no fields. */
+ NullPackageObject = 1 << 15,
+ /** The package contained a null attachment. */
+ NullPackageAttachment = 1 << 16,
+};
+
+ENUM_CLASS_FLAGS(CbValidateError);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Validate the compact binary data for one field in the view as specified by the mode flags.
+ *
+ * Only one top-level field is processed from the view, and validation recurses into any array or
+ * object within that field. To validate multiple consecutive top-level fields, call the function
+ * once for each top-level field. If the given view might contain multiple top-level fields, then
+ * either exclude the Padding flag from the Mode or use MeasureCompactBinary to break up the view
+ * into its constituent fields before validating.
+ *
+ * @param View A memory view containing at least one top-level field.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType);
+
+/**
+ * Validate the compact binary data for every field in the view as specified by the mode flags.
+ *
+ * This function expects the entire view to contain fields. Any trailing region of the view which
+ * does not contain a valid field will produce an OutOfBounds or InvalidType error instead of the
+ * Padding error that would be produced by the single field validation function.
+ *
+ * @see ValidateCompactBinary
+ */
+ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode);
+
+/**
+ * Validate the compact binary attachment pointed to by the view as specified by the mode flags.
+ *
+ * The attachment is validated with ValidateCompactBinary by using the validation mode specified.
+ * Include ECbValidateMode::Package to validate the attachment format and hash.
+ *
+ * @see ValidateCompactBinary
+ *
+ * @param View A memory view containing a package.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateObjectAttachment(MemoryView View, CbValidateMode Mode);
+
+/**
+ * Validate the compact binary package pointed to by the view as specified by the mode flags.
+ *
+ * The package, and attachments, are validated with ValidateCompactBinary by using the validation
+ * mode specified. Include ECbValidateMode::Package to validate the package format and hashes.
+ *
+ * @see ValidateCompactBinary
+ *
+ * @param View A memory view containing a package.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void usonvalidation_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/compactbinaryvalue.h b/src/zencore/include/zencore/compactbinaryvalue.h
new file mode 100644
index 000000000..0124a8983
--- /dev/null
+++ b/src/zencore/include/zencore/compactbinaryvalue.h
@@ -0,0 +1,290 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+#include <zencore/endian.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/memory.h>
+
+namespace zen {
+
+namespace CompactBinaryPrivate {
+
+ template<typename T>
+ static constexpr inline T ReadUnaligned(const void* const Memory)
+ {
+#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS
+ return *static_cast<const T*>(Memory);
+#else
+ T Value;
+ memcpy(&Value, Memory, sizeof(Value));
+ return Value;
+#endif
+ }
+} // namespace CompactBinaryPrivate
+/**
+ * A type that provides unchecked access to compact binary values.
+ *
+ * The main purpose of the type is to efficiently switch on field type. For every other use case,
+ * prefer to use the field, array, and object types directly. The accessors here do not check the
+ * type before reading the value, which means they can read out of bounds even on a valid compact
+ * binary value if the wrong accessor is used.
+ */
+class CbValue
+{
+public:
+ CbValue(CbFieldType Type, const void* Value);
+
+ CbObjectView AsObjectView() const;
+ CbArrayView AsArrayView() const;
+
+ MemoryView AsBinary() const;
+
+ /** Access as a string. Checks for range errors and uses the default if OutError is not null. */
+ std::string_view AsString(CbFieldError* OutError = nullptr, std::string_view Default = std::string_view()) const;
+
+ /** Access as a string as UTF8. Checks for range errors and uses the default if OutError is not null. */
+ std::u8string_view AsU8String(CbFieldError* OutError = nullptr, std::u8string_view Default = std::u8string_view()) const;
+
+ /**
+ * Access as an integer, with both positive and negative values returned as unsigned.
+ *
+ * Checks for range errors and uses the default if OutError is not null.
+ */
+ uint64_t AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError = nullptr, uint64_t Default = 0) const;
+
+ uint64_t AsIntegerPositive() const;
+ int64_t AsIntegerNegative() const;
+
+ float AsFloat32() const;
+ double AsFloat64() const;
+
+ bool AsBool() const;
+
+ inline IoHash AsObjectAttachment() const { return AsHash(); }
+ inline IoHash AsBinaryAttachment() const { return AsHash(); }
+ inline IoHash AsAttachment() const { return AsHash(); }
+
+ IoHash AsHash() const;
+ Guid AsUuid() const;
+
+ int64_t AsDateTimeTicks() const;
+ int64_t AsTimeSpanTicks() const;
+
+ Oid AsObjectId() const;
+
+ CbCustomById AsCustomById() const;
+ CbCustomByName AsCustomByName() const;
+
+ inline CbFieldType GetType() const { return Type; }
+ inline const void* GetData() const { return Data; }
+
+private:
+ const void* Data;
+ CbFieldType Type;
+};
+
+inline CbFieldView::CbFieldView(const CbValue& InValue) : Type(InValue.GetType()), Payload(InValue.GetData())
+{
+}
+
+inline CbValue
+CbFieldView::GetValue() const
+{
+ return CbValue(CbFieldTypeOps::GetType(Type), Payload);
+}
+
+inline CbValue::CbValue(CbFieldType InType, const void* InValue) : Data(InValue), Type(InType)
+{
+}
+
+inline CbObjectView
+CbValue::AsObjectView() const
+{
+ return CbObjectView(*this);
+}
+
+inline CbArrayView
+CbValue::AsArrayView() const
+{
+ return CbArrayView(*this);
+}
+
+inline MemoryView
+CbValue::AsBinary() const
+{
+ const uint8_t* const Bytes = static_cast<const uint8_t*>(Data);
+ uint32_t ValueSizeByteCount;
+ const uint64_t ValueSize = ReadVarUInt(Bytes, ValueSizeByteCount);
+ return MakeMemoryView(Bytes + ValueSizeByteCount, ValueSize);
+}
+
+inline std::string_view
+CbValue::AsString(CbFieldError* OutError, std::string_view Default) const
+{
+ const char* const Chars = static_cast<const char*>(Data);
+ uint32_t ValueSizeByteCount;
+ const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
+
+ if (OutError)
+ {
+ if (ValueSize >= (uint64_t(1) << 31))
+ {
+ *OutError = CbFieldError::RangeError;
+ return Default;
+ }
+ *OutError = CbFieldError::None;
+ }
+
+ return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+}
+
+inline std::u8string_view
+CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const
+{
+ const char8_t* const Chars = static_cast<const char8_t*>(Data);
+ uint32_t ValueSizeByteCount;
+ const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
+
+ if (OutError)
+ {
+ if (ValueSize >= (uint64_t(1) << 31))
+ {
+ *OutError = CbFieldError::RangeError;
+ return Default;
+ }
+ *OutError = CbFieldError::None;
+ }
+
+ return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+}
+
+inline uint64_t
+CbValue::AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError, uint64_t Default) const
+{
+ // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero.
+ const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1);
+ const uint64_t IsNegative = uint8_t(Type) & 1;
+
+ uint32_t MagnitudeByteCount;
+ const uint64_t Magnitude = ReadVarUInt(Data, MagnitudeByteCount);
+ const uint64_t Value = Magnitude ^ -int64_t(IsNegative);
+
+ if (OutError)
+ {
+ const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned);
+ *OutError = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
+
+ const uint64_t UseValueMask = -int64_t(IsInRange);
+ return (Value & UseValueMask) | (Default & ~UseValueMask);
+ }
+
+ return Value;
+}
+
+inline uint64_t
+CbValue::AsIntegerPositive() const
+{
+ uint32_t MagnitudeByteCount;
+ return ReadVarUInt(Data, MagnitudeByteCount);
+}
+
+inline int64_t
+CbValue::AsIntegerNegative() const
+{
+ uint32_t MagnitudeByteCount;
+ return int64_t(ReadVarUInt(Data, MagnitudeByteCount)) ^ -int64_t(1);
+}
+
+inline float
+CbValue::AsFloat32() const
+{
+ const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint32_t>(Data));
+ return reinterpret_cast<const float&>(Value);
+}
+
+inline double
+CbValue::AsFloat64() const
+{
+ const uint64_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint64_t>(Data));
+ return reinterpret_cast<const double&>(Value);
+}
+
+inline bool
+CbValue::AsBool() const
+{
+ return uint8_t(Type) & 1;
+}
+
+inline IoHash
+CbValue::AsHash() const
+{
+ return IoHash::MakeFrom(Data);
+}
+
+inline Guid
+CbValue::AsUuid() const
+{
+ Guid Value;
+ memcpy(&Value, Data, sizeof(Guid));
+ Value.A = FromNetworkOrder(Value.A);
+ Value.B = FromNetworkOrder(Value.B);
+ Value.C = FromNetworkOrder(Value.C);
+ Value.D = FromNetworkOrder(Value.D);
+ return Value;
+}
+
+inline int64_t
+CbValue::AsDateTimeTicks() const
+{
+ return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data));
+}
+
+inline int64_t
+CbValue::AsTimeSpanTicks() const
+{
+ return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data));
+}
+
+inline Oid
+CbValue::AsObjectId() const
+{
+ return Oid::FromMemory(Data);
+}
+
+inline CbCustomById
+CbValue::AsCustomById() const
+{
+ const uint8_t* Bytes = static_cast<const uint8_t*>(Data);
+ uint32_t DataSizeByteCount;
+ const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount);
+ Bytes += DataSizeByteCount;
+
+ CbCustomById Value;
+ uint32_t TypeIdByteCount;
+ Value.Id = ReadVarUInt(Bytes, TypeIdByteCount);
+ Value.Data = MakeMemoryView(Bytes + TypeIdByteCount, DataSize - TypeIdByteCount);
+ return Value;
+}
+
+inline CbCustomByName
+CbValue::AsCustomByName() const
+{
+ const uint8_t* Bytes = static_cast<const uint8_t*>(Data);
+ uint32_t DataSizeByteCount;
+ const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount);
+ Bytes += DataSizeByteCount;
+
+ uint32_t TypeNameLenByteCount;
+ const uint64_t TypeNameLen = ReadVarUInt(Bytes, TypeNameLenByteCount);
+ Bytes += TypeNameLenByteCount;
+
+ CbCustomByName Value;
+ Value.Name = std::u8string_view(reinterpret_cast<const char8_t*>(Bytes), static_cast<std::u8string_view::size_type>(TypeNameLen));
+ Value.Data = MakeMemoryView(Bytes + TypeNameLen, DataSize - TypeNameLen - TypeNameLenByteCount);
+ return Value;
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h
new file mode 100644
index 000000000..4e4b4d002
--- /dev/null
+++ b/src/zencore/include/zencore/compositebuffer.h
@@ -0,0 +1,142 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/sharedbuffer.h>
+#include <zencore/zencore.h>
+
+#include <functional>
+#include <span>
+#include <vector>
+
+namespace zen {
+
+/**
+ * CompositeBuffer is a non-contiguous buffer composed of zero or more immutable shared buffers.
+ *
+ * A composite buffer is most efficient when its segments are consumed as they are, but it can be
+ * flattened into a contiguous buffer, when necessary, by calling Flatten(). Ownership of segment
+ * buffers is not changed on construction, but if ownership of segments is required then that can
+ * be guaranteed by calling MakeOwned().
+ */
+
+class CompositeBuffer
+{
+public:
+ /**
+ * Construct a composite buffer by concatenating the buffers. Does not enforce ownership.
+ *
+ * Buffer parameters may be SharedBuffer, CompositeBuffer, or std::vector<SharedBuffer>.
+ */
+ template<typename... BufferTypes>
+ inline explicit CompositeBuffer(BufferTypes&&... Buffers)
+ {
+ if constexpr (sizeof...(Buffers) > 0)
+ {
+ m_Segments.reserve((GetBufferCount(std::forward<BufferTypes>(Buffers)) + ...));
+ (AppendBuffers(std::forward<BufferTypes>(Buffers)), ...);
+ std::erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); });
+ }
+ }
+
+ /** Reset this to null. */
+ ZENCORE_API void Reset();
+
+ /** Returns the total size of the composite buffer in bytes. */
+ [[nodiscard]] ZENCORE_API uint64_t GetSize() const;
+
+ /** Returns the segments that the buffer is composed from. */
+ [[nodiscard]] inline std::span<const SharedBuffer> GetSegments() const { return std::span<const SharedBuffer>{m_Segments}; }
+
+ /** Returns true if the composite buffer is not null. */
+ [[nodiscard]] inline explicit operator bool() const { return !IsNull(); }
+
+ /** Returns true if the composite buffer is null. */
+ [[nodiscard]] inline bool IsNull() const { return m_Segments.empty(); }
+
+ /** Returns true if every segment in the composite buffer is owned. */
+ [[nodiscard]] ZENCORE_API bool IsOwned() const;
+
+ /** Returns a copy of the buffer where every segment is owned. */
+ [[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() const&;
+ [[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() &&;
+
+ /** Returns the concatenation of the segments into a contiguous buffer. */
+ [[nodiscard]] ZENCORE_API SharedBuffer Flatten() const&;
+ [[nodiscard]] ZENCORE_API SharedBuffer Flatten() &&;
+
+ /** Returns the middle part of the buffer by taking the size starting at the offset. */
+ [[nodiscard]] ZENCORE_API CompositeBuffer Mid(uint64_t Offset, uint64_t Size = ~uint64_t(0)) const;
+
+ /**
+ * Returns a view of the range if contained by one segment, otherwise a view of a copy of the range.
+ *
+ * @note CopyBuffer is reused if large enough, and otherwise allocated when needed.
+ *
+ * @param Offset The byte offset in this buffer that the range starts at.
+ * @param Size The number of bytes in the range to view or copy.
+ * @param CopyBuffer The buffer to write the copy into if a copy is required.
+ */
+ [[nodiscard]] ZENCORE_API MemoryView ViewOrCopyRange(uint64_t Offset, uint64_t Size, UniqueBuffer& CopyBuffer) const;
+
+ /**
+ * Copies a range of the buffer to a contiguous region of memory.
+ *
+ * @param Target The view to copy to. Must be no larger than the data available at the offset.
+ * @param Offset The byte offset in this buffer to start copying from.
+ */
+ ZENCORE_API void CopyTo(MutableMemoryView Target, uint64_t Offset = 0) const;
+
+ /**
+ * Invokes a visitor with a view of each segment that intersects with a range.
+ *
+ * @param Offset The byte offset in this buffer to start visiting from.
+ * @param Size The number of bytes in the range to visit.
+ * @param Visitor The visitor to invoke from zero to GetSegments().Num() times.
+ */
+ ZENCORE_API void IterateRange(uint64_t Offset, uint64_t Size, std::function<void(MemoryView View)> Visitor) const;
+ ZENCORE_API void IterateRange(uint64_t Offset,
+ uint64_t Size,
+ std::function<void(MemoryView View, const SharedBuffer& ViewOuter)> Visitor) const;
+
+ struct Iterator
+ {
+ size_t SegmentIndex = 0;
+ uint64_t OffsetInSegment = 0;
+ };
+ ZENCORE_API Iterator GetIterator(uint64_t Offset) const;
+ ZENCORE_API MemoryView ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const;
+ ZENCORE_API void CopyTo(MutableMemoryView Target, Iterator& It) const;
+
+ /** A null composite buffer. */
+ static const CompositeBuffer Null;
+
+private:
+ static inline size_t GetBufferCount(const CompositeBuffer& Buffer) { return Buffer.m_Segments.size(); }
+ inline void AppendBuffers(const CompositeBuffer& Buffer)
+ {
+ m_Segments.insert(m_Segments.end(), begin(Buffer.m_Segments), end(Buffer.m_Segments));
+ }
+ inline void AppendBuffers(CompositeBuffer&& Buffer)
+ {
+ // TODO: this operates just like the by-reference version above
+ m_Segments.insert(m_Segments.end(), begin(Buffer.m_Segments), end(Buffer.m_Segments));
+ }
+
+ static inline size_t GetBufferCount(const SharedBuffer&) { return 1; }
+ inline void AppendBuffers(const SharedBuffer& Buffer) { m_Segments.push_back(Buffer); }
+ inline void AppendBuffers(SharedBuffer&& Buffer) { m_Segments.push_back(std::move(Buffer)); }
+
+ static inline size_t GetBufferCount(std::vector<SharedBuffer>&& Container) { return Container.size(); }
+ inline void AppendBuffers(std::vector<SharedBuffer>&& Container)
+ {
+ m_Segments.insert(m_Segments.end(), begin(Container), end(Container));
+ }
+
+private:
+ std::vector<SharedBuffer> m_Segments;
+};
+
+void compositebuffer_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h
new file mode 100644
index 000000000..99ce20d8a
--- /dev/null
+++ b/src/zencore/include/zencore/compress.h
@@ -0,0 +1,165 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore/zencore.h"
+
+#include "zencore/blake3.h"
+#include "zencore/compositebuffer.h"
+
+namespace zen {
+
+enum class OodleCompressor : uint8_t
+{
+ NotSet = 0,
+ Selkie = 1,
+ Mermaid = 2,
+ Kraken = 3,
+ Leviathan = 4,
+};
+
+enum class OodleCompressionLevel : int8_t
+{
+ HyperFast4 = -4,
+ HyperFast3 = -3,
+ HyperFast2 = -2,
+ HyperFast1 = -1,
+ None = 0,
+ SuperFast = 1,
+ VeryFast = 2,
+ Fast = 3,
+ Normal = 4,
+ Optimal1 = 5,
+ Optimal2 = 6,
+ Optimal3 = 7,
+ Optimal4 = 8,
+};
+
+/**
+ * A compressed buffer stores compressed data in a self-contained format.
+ *
+ * A buffer is self-contained in the sense that it can be decompressed without external knowledge
+ * of the compression format or the size of the raw data.
+ */
+class CompressedBuffer
+{
+public:
+ /**
+ * Compress the buffer using the specified compressor and compression level.
+ *
+ * Data that does not compress will be return uncompressed, as if with level None.
+ *
+ * @note Using a level of None will return a buffer that references owned raw data.
+ *
+ * @param RawData The raw data to be compressed.
+ * @param Compressor The compressor to encode with. May use NotSet if level is None.
+ * @param CompressionLevel The compression level to encode with.
+ * @param BlockSize The power-of-two block size to encode raw data in. 0 is default.
+ * @return An owned compressed buffer, or null on error.
+ */
+ [[nodiscard]] ZENCORE_API static CompressedBuffer Compress(const CompositeBuffer& RawData,
+ OodleCompressor Compressor = OodleCompressor::Mermaid,
+ OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
+ uint64_t BlockSize = 0);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer Compress(const SharedBuffer& RawData,
+ OodleCompressor Compressor = OodleCompressor::Mermaid,
+ OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
+ uint64_t BlockSize = 0);
+
+ /**
+ * Construct from a compressed buffer previously created by Compress().
+ *
+ * @return A compressed buffer, or null on error, such as an invalid format or corrupt header.
+ */
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData);
+ [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
+ [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(const IoBuffer& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
+
+ /** Reset this to null. */
+ inline void Reset() { CompressedData.Reset(); }
+
+ /** Returns true if the compressed buffer is not null. */
+ [[nodiscard]] inline explicit operator bool() const { return !IsNull(); }
+
+ /** Returns true if the compressed buffer is null. */
+ [[nodiscard]] inline bool IsNull() const { return CompressedData.IsNull(); }
+
+ /** Returns true if the composite buffer is owned. */
+ [[nodiscard]] inline bool IsOwned() const { return CompressedData.IsOwned(); }
+
+ /** Returns a copy of the compressed buffer that owns its underlying memory. */
+ [[nodiscard]] inline CompressedBuffer MakeOwned() const& { return FromCompressedNoValidate(CompressedData.MakeOwned()); }
+ [[nodiscard]] inline CompressedBuffer MakeOwned() && { return FromCompressedNoValidate(std::move(CompressedData).MakeOwned()); }
+
+ /** Returns a composite buffer containing the compressed data. May be null. May not be owned. */
+ [[nodiscard]] inline const CompositeBuffer& GetCompressed() const& { return CompressedData; }
+ [[nodiscard]] inline CompositeBuffer GetCompressed() && { return std::move(CompressedData); }
+
+ /** Returns the size of the compressed data. Zero if this is null. */
+ [[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); }
+
+ /** Returns the size of the raw data. Zero on error or if this is empty or null. */
+ [[nodiscard]] ZENCORE_API uint64_t DecodeRawSize() const;
+
+ /** Returns the hash of the raw data. Zero on error or if this is null. */
+ [[nodiscard]] ZENCORE_API IoHash DecodeRawHash() const;
+
+ [[nodiscard]] ZENCORE_API CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const;
+
+ /**
+ * Returns the compressor and compression level used by this buffer.
+ *
+ * The compressor and compression level may differ from those specified when creating the buffer
+ * because an incompressible buffer is stored with no compression. Parameters cannot be accessed
+ * if this is null or uses a method other than Oodle, in which case this returns false.
+ *
+ * @return True if parameters were written, otherwise false.
+ */
+ [[nodiscard]] ZENCORE_API bool TryGetCompressParameters(OodleCompressor& OutCompressor,
+ OodleCompressionLevel& OutCompressionLevel,
+ uint64_t& OutBlockSize) const;
+
+ /**
+ * Decompress into a memory view that is less or equal GetRawSize() bytes.
+ */
+ [[nodiscard]] ZENCORE_API bool TryDecompressTo(MutableMemoryView RawView, uint64_t RawOffset = 0) const;
+
+ /**
+ * Decompress into an owned buffer.
+ *
+ * @return An owned buffer containing the raw data, or null on error.
+ */
+ [[nodiscard]] ZENCORE_API SharedBuffer Decompress(uint64_t RawOffset = 0, uint64_t RawSize = ~uint64_t(0)) const;
+
+ /**
+ * Decompress into an owned composite buffer.
+ *
+ * @return An owned buffer containing the raw data, or null on error.
+ */
+ [[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const;
+
+ /** A null compressed buffer. */
+ static const CompressedBuffer Null;
+
+private:
+ CompositeBuffer CompressedData;
+};
+
+void compress_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/config.h.in b/src/zencore/include/zencore/config.h.in
new file mode 100644
index 000000000..3372eca2a
--- /dev/null
+++ b/src/zencore/include/zencore/config.h.in
@@ -0,0 +1,16 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+// NOTE: Generated from config.h.in
+
+#define ZEN_CFG_VERSION "${VERSION}"
+#define ZEN_CFG_VERSION_MAJOR ${VERSION_MAJOR}
+#define ZEN_CFG_VERSION_MINOR ${VERSION_MINOR}
+#define ZEN_CFG_VERSION_ALTER ${VERSION_ALTER}
+#define ZEN_CFG_VERSION_BUILD ${VERSION_BUILD}
+#define ZEN_CFG_VERSION_BRANCH "${GIT_BRANCH}"
+#define ZEN_CFG_VERSION_COMMIT "${GIT_COMMIT}"
+#define ZEN_CFG_VERSION_BUILD_STRING "${VERSION}-${plat}-${arch}-${mode}"
+#define ZEN_CFG_VERSION_BUILD_STRING_FULL "${VERSION}-${VERSION_BUILD}-${plat}-${arch}-${mode}-${GIT_COMMIT}"
+#define ZEN_CFG_SCHEMA_VERSION ${ZEN_SCHEMA_VERSION}
diff --git a/src/zencore/include/zencore/crc32.h b/src/zencore/include/zencore/crc32.h
new file mode 100644
index 000000000..336bda77e
--- /dev/null
+++ b/src/zencore/include/zencore/crc32.h
@@ -0,0 +1,13 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+namespace zen {
+
+uint32_t MemCrc32(const void* InData, size_t Length, uint32_t Crc = 0);
+uint32_t MemCrc32_Deprecated(const void* InData, size_t Length, uint32_t Crc = 0);
+uint32_t StrCrc_Deprecated(const char* Data);
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/crypto.h b/src/zencore/include/zencore/crypto.h
new file mode 100644
index 000000000..83d416b0f
--- /dev/null
+++ b/src/zencore/include/zencore/crypto.h
@@ -0,0 +1,77 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/memory.h>
+#include <zencore/zencore.h>
+
+#include <memory>
+#include <optional>
+
+namespace zen {
+
+template<size_t BitCount>
+struct CryptoBits
+{
+public:
+ static constexpr size_t ByteCount = BitCount / 8;
+
+ CryptoBits() = default;
+
+ bool IsNull() const { return memcmp(&m_Bits, &Zero, ByteCount) == 0; }
+ bool IsValid() const { return IsNull() == false; }
+
+ size_t GetSize() const { return ByteCount; }
+ size_t GetBitCount() const { return BitCount; }
+
+ MemoryView GetView() const { return MemoryView(m_Bits, ByteCount); }
+
+ static CryptoBits FromMemoryView(MemoryView Bits)
+ {
+ if (Bits.GetSize() != ByteCount)
+ {
+ return CryptoBits();
+ }
+
+ return CryptoBits(Bits);
+ }
+
+ static CryptoBits FromString(std::string_view Str) { return FromMemoryView(MakeMemoryView(Str)); }
+
+private:
+ CryptoBits(MemoryView Bits)
+ {
+ ZEN_ASSERT(Bits.GetSize() == GetSize());
+ memcpy(&m_Bits, Bits.GetData(), GetSize());
+ }
+
+ static constexpr uint8_t Zero[ByteCount] = {0};
+
+ uint8_t m_Bits[ByteCount] = {0};
+};
+
+using AesKey256Bit = CryptoBits<256>;
+using AesIV128Bit = CryptoBits<128>;
+
+class Aes
+{
+public:
+ static constexpr size_t BlockSize = 16;
+
+ static MemoryView Encrypt(const AesKey256Bit& Key,
+ const AesIV128Bit& IV,
+ MemoryView In,
+ MutableMemoryView Out,
+ std::optional<std::string>& Reason);
+
+ static MemoryView Decrypt(const AesKey256Bit& Key,
+ const AesIV128Bit& IV,
+ MemoryView In,
+ MutableMemoryView Out,
+ std::optional<std::string>& Reason);
+};
+
+void crypto_forcelink();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/endian.h b/src/zencore/include/zencore/endian.h
new file mode 100644
index 000000000..7a9e6b44c
--- /dev/null
+++ b/src/zencore/include/zencore/endian.h
@@ -0,0 +1,113 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <cstdint>
+
+namespace zen {
+
+inline uint16_t
+ByteSwap(uint16_t x)
+{
+#if ZEN_COMPILER_MSC
+ return _byteswap_ushort(x);
+#else
+ return __builtin_bswap16(x);
+#endif
+}
+
+inline uint32_t
+ByteSwap(uint32_t x)
+{
+#if ZEN_COMPILER_MSC
+ return _byteswap_ulong(x);
+#else
+ return __builtin_bswap32(x);
+#endif
+}
+
+inline uint64_t
+ByteSwap(uint64_t x)
+{
+#if ZEN_COMPILER_MSC
+ return _byteswap_uint64(x);
+#else
+ return __builtin_bswap64(x);
+#endif
+}
+
+inline uint16_t
+FromNetworkOrder(uint16_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint32_t
+FromNetworkOrder(uint32_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint64_t
+FromNetworkOrder(uint64_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint16_t
+FromNetworkOrder(int16_t x)
+{
+ return ByteSwap(uint16_t(x));
+}
+
+inline uint32_t
+FromNetworkOrder(int32_t x)
+{
+ return ByteSwap(uint32_t(x));
+}
+
+inline uint64_t
+FromNetworkOrder(int64_t x)
+{
+ return ByteSwap(uint64_t(x));
+}
+
+inline uint16_t
+ToNetworkOrder(uint16_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint32_t
+ToNetworkOrder(uint32_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint64_t
+ToNetworkOrder(uint64_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint16_t
+ToNetworkOrder(int16_t x)
+{
+ return ByteSwap(uint16_t(x));
+}
+
+inline uint32_t
+ToNetworkOrder(int32_t x)
+{
+ return ByteSwap(uint32_t(x));
+}
+
+inline uint64_t
+ToNetworkOrder(int64_t x)
+{
+ return ByteSwap(uint64_t(x));
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/enumflags.h b/src/zencore/include/zencore/enumflags.h
new file mode 100644
index 000000000..ebe747bf0
--- /dev/null
+++ b/src/zencore/include/zencore/enumflags.h
@@ -0,0 +1,61 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+// Enum class helpers
+
+// Defines all bitwise operators for enum classes so it can be (mostly) used as a regular flags enum
+#define ENUM_CLASS_FLAGS(Enum) \
+ inline Enum& operator|=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \
+ inline Enum& operator&=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \
+ inline Enum& operator^=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator|(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator&(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator^(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \
+ inline constexpr bool operator!(Enum E) { return !(__underlying_type(Enum))E; } \
+ inline constexpr Enum operator~(Enum E) { return (Enum) ~(__underlying_type(Enum))E; }
+
+// Friends all bitwise operators for enum classes so the definition can be kept private / protected.
+#define FRIEND_ENUM_CLASS_FLAGS(Enum) \
+ friend Enum& operator|=(Enum& Lhs, Enum Rhs); \
+ friend Enum& operator&=(Enum& Lhs, Enum Rhs); \
+ friend Enum& operator^=(Enum& Lhs, Enum Rhs); \
+ friend constexpr Enum operator|(Enum Lhs, Enum Rhs); \
+ friend constexpr Enum operator&(Enum Lhs, Enum Rhs); \
+ friend constexpr Enum operator^(Enum Lhs, Enum Rhs); \
+ friend constexpr bool operator!(Enum E); \
+ friend constexpr Enum operator~(Enum E);
+
+template<typename Enum>
+constexpr bool
+EnumHasAllFlags(Enum Flags, Enum Contains)
+{
+ return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) == ((__underlying_type(Enum))Contains);
+}
+
+template<typename Enum>
+constexpr bool
+EnumHasAnyFlags(Enum Flags, Enum Contains)
+{
+ return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) != 0;
+}
+
+template<typename Enum>
+void
+EnumAddFlags(Enum& Flags, Enum FlagsToAdd)
+{
+ Flags |= FlagsToAdd;
+}
+
+template<typename Enum>
+void
+EnumRemoveFlags(Enum& Flags, Enum FlagsToRemove)
+{
+ Flags &= ~FlagsToRemove;
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/except.h b/src/zencore/include/zencore/except.h
new file mode 100644
index 000000000..c61db5ba9
--- /dev/null
+++ b/src/zencore/include/zencore/except.h
@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/string.h>
+#if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+#else
+# include <errno.h>
+#endif
+#if __has_include("source_location")
+# include <source_location>
+#endif
+#include <string>
+#include <system_error>
+
+namespace zen {
+
+#if ZEN_PLATFORM_WINDOWS
+ZENCORE_API void ThrowSystemException [[noreturn]] (HRESULT hRes, std::string_view Message);
+#endif // ZEN_PLATFORM_WINDOWS
+
+#if defined(__cpp_lib_source_location)
+ZENCORE_API void ThrowLastErrorImpl [[noreturn]] (std::string_view Message, const std::source_location& Location);
+# define ThrowLastError(Message) ThrowLastErrorImpl(Message, std::source_location::current())
+#else
+ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message);
+#endif
+
+ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message);
+
+ZENCORE_API std::string GetLastErrorAsString();
+ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode);
+
+inline int32_t
+GetLastError()
+{
+#if ZEN_PLATFORM_WINDOWS
+ return ::GetLastError();
+#else
+ return errno;
+#endif
+}
+
+inline std::error_code
+MakeErrorCode(uint32_t ErrorCode) noexcept
+{
+ return std::error_code(ErrorCode, std::system_category());
+}
+
+inline std::error_code
+MakeErrorCodeFromLastError() noexcept
+{
+ return std::error_code(zen::GetLastError(), std::system_category());
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
new file mode 100644
index 000000000..fa5f94170
--- /dev/null
+++ b/src/zencore/include/zencore/filesystem.h
@@ -0,0 +1,190 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/iobuffer.h>
+#include <zencore/string.h>
+
+#include <filesystem>
+#include <functional>
+
+namespace zen {
+
+class IoBuffer;
+
+/** Delete directory (after deleting any contents)
+ */
+ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir);
+
+/** Ensure directory exists.
+
+ Will also create any required parent directories
+ */
+ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir);
+
+/** Ensure directory exists and delete contents (if any) before returning
+ */
+ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir);
+
+/** Map native file handle to a path
+ */
+ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle);
+
+ZENCORE_API std::filesystem::path GetRunningExecutablePath();
+
+/** Set the max open file handle count to max allowed for the current process on Linux and MacOS
+ */
+ZENCORE_API void MaximizeOpenFileCount();
+
+struct FileContents
+{
+ std::vector<IoBuffer> Data;
+ std::error_code ErrorCode;
+
+ IoBuffer Flatten();
+};
+
+ZENCORE_API FileContents ReadStdIn();
+ZENCORE_API FileContents ReadFile(std::filesystem::path Path);
+ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc);
+ZENCORE_API void WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount);
+ZENCORE_API void WriteFile(std::filesystem::path Path, IoBuffer Data);
+
+struct CopyFileOptions
+{
+ bool EnableClone = true;
+ bool MustClone = false;
+};
+
+ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
+ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path);
+
+ZENCORE_API void PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out);
+ZENCORE_API std::string PathToUtf8(const std::filesystem::path& Path);
+
+extern template class StringBuilderImpl<std::filesystem::path::value_type>;
+
+/**
+ * Helper class for building paths. Backed by a string builder.
+ *
+ */
+class PathBuilderBase : public StringBuilderImpl<std::filesystem::path::value_type>
+{
+private:
+ using Super = StringBuilderImpl<std::filesystem::path::value_type>;
+
+protected:
+ using CharType = std::filesystem::path::value_type;
+ using ViewType = std::basic_string_view<CharType>;
+
+public:
+ void Append(const std::filesystem::path& Rhs) { Super::Append(Rhs.c_str()); }
+ void operator/=(const std::filesystem::path& Rhs) { this->operator/=(Rhs.c_str()); };
+ void operator/=(const CharType* Rhs)
+ {
+ AppendSeparator();
+ Super::Append(Rhs);
+ }
+ operator ViewType() const { return ToView(); }
+ std::basic_string_view<CharType> ToView() const { return std::basic_string_view<CharType>(Data(), Size()); }
+ std::filesystem::path ToPath() const { return std::filesystem::path(ToView()); }
+
+ std::string ToUtf8() const
+ {
+#if ZEN_PLATFORM_WINDOWS
+ return WideToUtf8(ToView());
+#else
+ return std::string(ToView());
+#endif
+ }
+
+ void AppendSeparator()
+ {
+ if (ToView().ends_with(std::filesystem::path::preferred_separator)
+#if ZEN_PLATFORM_WINDOWS
+ || ToView().ends_with('/')
+#endif
+ )
+ return;
+
+ Super::Append(std::filesystem::path::preferred_separator);
+ }
+};
+
+template<size_t N>
+class PathBuilder : public PathBuilderBase
+{
+public:
+ PathBuilder() { Init(m_Buffer, N); }
+
+private:
+ PathBuilderBase::CharType m_Buffer[N];
+};
+
+template<size_t N>
+class ExtendablePathBuilder : public PathBuilder<N>
+{
+public:
+ ExtendablePathBuilder() { this->m_IsExtendable = true; }
+};
+
+struct DiskSpace
+{
+ uint64_t Free{};
+ uint64_t Total{};
+};
+
+ZENCORE_API DiskSpace DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error);
+
+inline bool
+DiskSpaceInfo(std::filesystem::path Directory, DiskSpace& Space)
+{
+ std::error_code Err;
+ Space = DiskSpaceInfo(Directory, Err);
+ return !Err;
+}
+
+/**
+ * Efficient file system traversal
+ *
+ * Uses the best available mechanism for the platform in question and could take
+ * advantage of any file system tracking mechanisms in the future
+ *
+ */
+class FileSystemTraversal
+{
+public:
+ struct TreeVisitor
+ {
+ using path_view = std::basic_string_view<std::filesystem::path::value_type>;
+ using path_string = std::filesystem::path::string_type;
+
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0;
+
+ // This should return true if we should recurse into the directory
+ virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0;
+ };
+
+ void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
+};
+
+struct DirectoryContent
+{
+ static const uint8_t IncludeDirsFlag = 1u << 0;
+ static const uint8_t IncludeFilesFlag = 1u << 1;
+ static const uint8_t RecursiveFlag = 1u << 2;
+ std::vector<std::filesystem::path> Files;
+ std::vector<std::filesystem::path> Directories;
+};
+
+void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent);
+
+std::string GetEnvVariable(std::string_view VariableName);
+
+//////////////////////////////////////////////////////////////////////////
+
+void filesystem_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/fmtutils.h b/src/zencore/include/zencore/fmtutils.h
new file mode 100644
index 000000000..70867fe72
--- /dev/null
+++ b/src/zencore/include/zencore/fmtutils.h
@@ -0,0 +1,52 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <filesystem>
+#include <string_view>
+
+// Custom formatting for some zencore types
+
+template<>
+struct fmt::formatter<zen::IoHash> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const zen::IoHash& Hash, FormatContext& ctx)
+ {
+ zen::IoHash::String_t String;
+ Hash.ToHexString(String);
+ return formatter<string_view>::format({String, zen::IoHash::StringLength}, ctx);
+ }
+};
+
+template<>
+struct fmt::formatter<zen::Oid> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const zen::Oid& Id, FormatContext& ctx)
+ {
+ zen::StringBuilder<32> String;
+ Id.ToString(String);
+ return formatter<string_view>::format({String.c_str(), zen::Oid::StringLength}, ctx);
+ }
+};
+
+template<>
+struct fmt::formatter<std::filesystem::path> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const std::filesystem::path& Path, FormatContext& ctx)
+ {
+ zen::ExtendableStringBuilder<128> String;
+ String << Path.u8string();
+ return formatter<string_view>::format(String.ToView(), ctx);
+ }
+};
diff --git a/src/zencore/include/zencore/intmath.h b/src/zencore/include/zencore/intmath.h
new file mode 100644
index 000000000..f24caed6e
--- /dev/null
+++ b/src/zencore/include/zencore/intmath.h
@@ -0,0 +1,183 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <stdint.h>
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_COMPILER_MSC || ZEN_PLATFORM_WINDOWS
+# pragma intrinsic(_BitScanReverse)
+# pragma intrinsic(_BitScanReverse64)
+#else
+inline uint8_t
+_BitScanReverse(unsigned long* Index, uint32_t Mask)
+{
+ if (Mask == 0)
+ {
+ return 0;
+ }
+
+ *Index = 31 - __builtin_clz(Mask);
+ return 1;
+}
+
+inline uint8_t
+_BitScanReverse64(unsigned long* Index, uint64_t Mask)
+{
+ if (Mask == 0)
+ {
+ return 0;
+ }
+
+ *Index = 63 - __builtin_clzll(Mask);
+ return 1;
+}
+
+inline uint8_t
+_BitScanForward64(unsigned long* Index, uint64_t Mask)
+{
+ if (Mask == 0)
+ {
+ return 0;
+ }
+
+ *Index = __builtin_ctzll(Mask);
+ return 1;
+}
+#endif
+
+namespace zen {
+
+inline constexpr bool
+IsPow2(uint64_t n)
+{
+ return 0 == (n & (n - 1));
+}
+
+/// Round an integer up to the closest integer multiplier of 'base' ('base' must be a power of two)
+template<Integral T>
+T
+RoundUp(T Value, auto Base)
+{
+ ZEN_ASSERT_SLOW(IsPow2(Base));
+ return ((Value + T(Base - 1)) & (~T(Base - 1)));
+}
+
+bool
+IsMultipleOf(Integral auto Value, auto MultiplierPow2)
+{
+ ZEN_ASSERT_SLOW(IsPow2(MultiplierPow2));
+ return (Value & (MultiplierPow2 - 1)) == 0;
+}
+
+inline uint64_t
+NextPow2(uint64_t n)
+{
+ // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+
+ --n;
+
+ n |= n >> 1;
+ n |= n >> 2;
+ n |= n >> 4;
+ n |= n >> 8;
+ n |= n >> 16;
+ n |= n >> 32;
+
+ return n + 1;
+}
+
+static inline uint32_t
+FloorLog2(uint32_t Value)
+{
+ // Use BSR to return the log2 of the integer
+ unsigned long Log2;
+ if (_BitScanReverse(&Log2, Value) != 0)
+ {
+ return Log2;
+ }
+
+ return 0;
+}
+
+static inline uint32_t
+CountLeadingZeros(uint32_t Value)
+{
+ unsigned long Log2 = 0;
+ _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1);
+ return 32 - Log2;
+}
+
+static inline uint64_t
+FloorLog2_64(uint64_t Value)
+{
+ unsigned long Log2 = 0;
+ long Mask = -long(_BitScanReverse64(&Log2, Value) != 0);
+ return Log2 & Mask;
+}
+
+static inline uint64_t
+CountLeadingZeros64(uint64_t Value)
+{
+ unsigned long Log2 = 0;
+ long Mask = -long(_BitScanReverse64(&Log2, Value) != 0);
+ return ((63 - Log2) & Mask) | (64 & ~Mask);
+}
+
+static inline uint64_t
+CeilLogTwo64(uint64_t Arg)
+{
+ int64_t Bitmask = ((int64_t)(CountLeadingZeros64(Arg) << 57)) >> 63;
+ return (64 - CountLeadingZeros64(Arg - 1)) & (~Bitmask);
+}
+
+static inline uint64_t
+CountTrailingZeros64(uint64_t Value)
+{
+ if (Value == 0)
+ {
+ return 64;
+ }
+ unsigned long BitIndex; // 0-based, where the LSB is 0 and MSB is 31
+ _BitScanForward64(&BitIndex, Value); // Scans from LSB to MSB
+ return BitIndex;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+static inline bool
+IsPointerAligned(const void* Ptr, uint64_t Alignment)
+{
+ ZEN_ASSERT_SLOW(IsPow2(Alignment));
+
+ return 0 == (reinterpret_cast<uintptr_t>(Ptr) & (Alignment - 1));
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_PLATFORM_WINDOWS
+# ifdef min
+# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead"
+# endif
+#endif
+
+constexpr auto
+Min(auto x, auto y)
+{
+ return x < y ? x : y;
+}
+
+constexpr auto
+Max(auto x, auto y)
+{
+ return x > y ? x : y;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void intmath_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h
new file mode 100644
index 000000000..a39dbf6d6
--- /dev/null
+++ b/src/zencore/include/zencore/iobuffer.h
@@ -0,0 +1,423 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <memory.h>
+#include <zencore/memory.h>
+#include <atomic>
+#include "refcount.h"
+#include "zencore.h"
+
+#include <filesystem>
+
+namespace zen {
+
+struct IoHash;
+struct IoBufferExtendedCore;
+
+enum class ZenContentType : uint8_t
+{
+ kBinary = 0, // Note that since this is zero, this will be the default value in IoBuffer
+ kText = 1,
+ kJSON = 2,
+ kCbObject = 3,
+ kCbPackage = 4,
+ kYAML = 5,
+ kCbPackageOffer = 6,
+ kCompressedBinary = 7,
+ kUnknownContentType = 8,
+ kHTML = 9,
+ kJavaScript = 10,
+ kCSS = 11,
+ kPNG = 12,
+ kIcon = 13,
+ kCOUNT
+};
+
+inline std::string_view
+ToString(ZenContentType ContentType)
+{
+ using namespace std::literals;
+
+ switch (ContentType)
+ {
+ default:
+ case ZenContentType::kUnknownContentType:
+ return "unknown"sv;
+ case ZenContentType::kBinary:
+ return "binary"sv;
+ case ZenContentType::kText:
+ return "text"sv;
+ case ZenContentType::kJSON:
+ return "json"sv;
+ case ZenContentType::kCbObject:
+ return "cb-object"sv;
+ case ZenContentType::kCbPackage:
+ return "cb-package"sv;
+ case ZenContentType::kCbPackageOffer:
+ return "cb-package-offer"sv;
+ case ZenContentType::kCompressedBinary:
+ return "compressed-binary"sv;
+ case ZenContentType::kYAML:
+ return "yaml"sv;
+ case ZenContentType::kHTML:
+ return "html"sv;
+ case ZenContentType::kJavaScript:
+ return "javascript"sv;
+ case ZenContentType::kCSS:
+ return "css"sv;
+ case ZenContentType::kPNG:
+ return "png"sv;
+ case ZenContentType::kIcon:
+ return "icon"sv;
+ }
+}
+
+struct IoBufferFileReference
+{
+ void* FileHandle;
+ uint64_t FileChunkOffset;
+ uint64_t FileChunkSize;
+};
+
+struct IoBufferCore
+{
+public:
+ inline IoBufferCore() : m_Flags(kIsNull) {}
+ inline IoBufferCore(const void* DataPtr, size_t SizeBytes) : m_DataPtr(DataPtr), m_DataBytes(SizeBytes) {}
+ inline IoBufferCore(const IoBufferCore* Outer, const void* DataPtr, size_t SizeBytes)
+ : m_DataPtr(DataPtr)
+ , m_DataBytes(SizeBytes)
+ , m_OuterCore(Outer)
+ {
+ }
+
+ ZENCORE_API explicit IoBufferCore(size_t SizeBytes);
+ ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment);
+ ZENCORE_API ~IoBufferCore();
+
+ // Reference counting
+
+ inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); }
+ inline uint32_t Release() const
+ {
+ const uint32_t NewRefCount = AtomicDecrement(const_cast<IoBufferCore*>(this)->m_RefCount);
+ if (NewRefCount == 0)
+ {
+ DeleteThis();
+ }
+ return NewRefCount;
+ }
+
+ // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it
+
+ IoBufferCore(const IoBufferCore&) = delete;
+ IoBufferCore(IoBufferCore&&) = delete;
+ IoBufferCore& operator=(const IoBufferCore&) = delete;
+ IoBufferCore& operator=(IoBufferCore&&) = delete;
+
+ //
+
+ ZENCORE_API void Materialize() const;
+ ZENCORE_API void DeleteThis() const;
+ ZENCORE_API void MakeOwned(bool Immutable = true);
+
+ inline void EnsureDataValid() const
+ {
+ const uint32_t LocalFlags = m_Flags.load(std::memory_order_acquire);
+ if ((LocalFlags & kIsExtended) && !(LocalFlags & kIsMaterialized))
+ {
+ Materialize();
+ }
+ }
+
+ inline bool IsOwnedByThis() const { return !!(m_Flags.load(std::memory_order_relaxed) & kIsOwnedByThis); }
+
+ inline void SetIsOwnedByThis(bool NewState)
+ {
+ if (NewState)
+ {
+ m_Flags.fetch_or(kIsOwnedByThis, std::memory_order_relaxed);
+ }
+ else
+ {
+ m_Flags.fetch_and(~kIsOwnedByThis, std::memory_order_relaxed);
+ }
+ }
+
+ inline bool IsOwned() const
+ {
+ if (IsOwnedByThis())
+ {
+ return true;
+ }
+ return m_OuterCore && m_OuterCore->IsOwned();
+ }
+
+ inline bool IsImmutable() const { return (m_Flags.load(std::memory_order_relaxed) & kIsMutable) == 0; }
+ inline bool IsWholeFile() const { return (m_Flags.load(std::memory_order_relaxed) & kIsWholeFile) != 0; }
+ inline bool IsNull() const { return (m_Flags.load(std::memory_order_relaxed) & kIsNull) != 0; }
+
+ inline IoBufferExtendedCore* ExtendedCore();
+ inline const IoBufferExtendedCore* ExtendedCore() const;
+
+ ZENCORE_API void* MutableDataPointer() const;
+
+ inline const void* DataPointer() const
+ {
+ EnsureDataValid();
+ return m_DataPtr;
+ }
+
+ inline size_t DataBytes() const { return m_DataBytes; }
+
+ inline void Set(const void* Ptr, size_t Sz)
+ {
+ m_DataPtr = Ptr;
+ m_DataBytes = Sz;
+ }
+
+ inline void SetIsImmutable(bool NewState)
+ {
+ if (!NewState)
+ {
+ m_Flags.fetch_or(kIsMutable, std::memory_order_relaxed);
+ }
+ else
+ {
+ m_Flags.fetch_and(~kIsMutable, std::memory_order_relaxed);
+ }
+ }
+
+ inline void SetIsWholeFile(bool NewState)
+ {
+ if (NewState)
+ {
+ m_Flags.fetch_or(kIsWholeFile, std::memory_order_relaxed);
+ }
+ else
+ {
+ m_Flags.fetch_and(~kIsWholeFile, std::memory_order_relaxed);
+ }
+ }
+
+ inline void SetContentType(ZenContentType ContentType)
+ {
+ ZEN_ASSERT_SLOW((uint32_t(ContentType) & kContentTypeMask) == uint32_t(ContentType));
+ uint32_t OldValue = m_Flags.load(std::memory_order_relaxed);
+ uint32_t NewValue;
+ do
+ {
+ NewValue = (OldValue & ~(kContentTypeMask << kContentTypeShift)) | (uint32_t(ContentType) << kContentTypeShift);
+ } while (!m_Flags.compare_exchange_weak(OldValue, NewValue, std::memory_order_relaxed, std::memory_order_relaxed));
+ }
+
+ inline ZenContentType GetContentType() const
+ {
+ return ZenContentType((m_Flags.load(std::memory_order_relaxed) >> kContentTypeShift) & kContentTypeMask);
+ }
+
+ inline uint32_t GetRefCount() const { return m_RefCount; }
+
+protected:
+ uint32_t m_RefCount = 0;
+ mutable std::atomic<uint32_t> m_Flags{0};
+ mutable const void* m_DataPtr = nullptr;
+ size_t m_DataBytes = 0;
+ RefPtr<const IoBufferCore> m_OuterCore;
+
+ enum
+ {
+ kContentTypeShift = 24,
+ kContentTypeMask = 0xf
+ };
+
+ static_assert((uint32_t(ZenContentType::kUnknownContentType) & ~kContentTypeMask) == 0);
+
+ enum Flags : uint32_t
+ {
+ kIsNull = 1 << 0, // This is a null IoBuffer
+ kIsMutable = 1 << 1,
+ kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore
+ kIsMaterialized = 1 << 3, // Data pointers are valid
+ kLowLevelAlloc = 1 << 4, // Using direct memory allocation
+ kIsWholeFile = 1 << 5, // References an entire file
+ kIoBufferAlloc = 1 << 6, // Using IoBuffer allocator
+ kIsOwnedByThis = 1 << 7,
+
+ // Note that we have some extended flags defined below
+ // so not all bits are available to use here
+
+ kContentTypeBit0 = 1 << (24 + 0), // These constants
+ kContentTypeBit1 = 1 << (24 + 1), // are here mostly to
+ kContentTypeBit2 = 1 << (24 + 2), // indicate that these
+ kContentTypeBit3 = 1 << (24 + 3), // bits are reserved
+ };
+
+ void AllocateBuffer(size_t InSize, size_t Alignment) const;
+ void FreeBuffer();
+};
+
+/**
+ * An "Extended" core references a segment of a file
+ */
+
+struct IoBufferExtendedCore : public IoBufferCore
+{
+ IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership);
+ IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size);
+ ~IoBufferExtendedCore();
+
+ enum ExtendedFlags
+ {
+ kOwnsFile = 1 << 16,
+ kOwnsMmap = 1 << 17
+ };
+
+ void Materialize() const;
+ bool GetFileReference(IoBufferFileReference& OutRef) const;
+ void MarkAsDeleteOnClose();
+
+private:
+ void* m_FileHandle = nullptr;
+ uint64_t m_FileOffset = 0;
+ mutable void* m_MmapHandle = nullptr;
+ mutable void* m_MappedPointer = nullptr;
+ bool m_DeleteOnClose = false;
+};
+
+inline IoBufferExtendedCore*
+IoBufferCore::ExtendedCore()
+{
+ if (m_Flags.load(std::memory_order_relaxed) & kIsExtended)
+ {
+ return static_cast<IoBufferExtendedCore*>(this);
+ }
+
+ return nullptr;
+}
+
+inline const IoBufferExtendedCore*
+IoBufferCore::ExtendedCore() const
+{
+ if (m_Flags.load(std::memory_order_relaxed) & kIsExtended)
+ {
+ return static_cast<const IoBufferExtendedCore*>(this);
+ }
+
+ return nullptr;
+}
+
+/**
+ * I/O buffer
+ *
+ * This represents a reference to a payload in memory or on disk
+ *
+ */
+class IoBuffer
+{
+public:
+ enum ECloneTag
+ {
+ Clone
+ };
+ enum EWrapTag
+ {
+ Wrap
+ };
+ enum EFileTag
+ {
+ File
+ };
+ enum EBorrowedFileTag
+ {
+ BorrowedFile
+ };
+
+ inline IoBuffer() = default;
+ inline IoBuffer(IoBuffer&& Rhs) noexcept = default;
+ inline IoBuffer(const IoBuffer& Rhs) = default;
+ inline IoBuffer& operator=(const IoBuffer& Rhs) = default;
+ inline IoBuffer& operator=(IoBuffer&& Rhs) noexcept = default;
+
+ /** Create an uninitialized buffer of the given size
+ */
+ ZENCORE_API explicit IoBuffer(size_t InSize);
+
+ /** Create an uninitialized buffer of the given size with the specified alignment
+ */
+ ZENCORE_API explicit IoBuffer(size_t InSize, uint64_t InAlignment);
+
+ /** Create a buffer which references a sequence of bytes inside another buffer
+ */
+ ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes = ~0ull);
+
+ /** Create a buffer which references a range of bytes which we assume will live
+ * for the entire life time.
+ */
+ inline IoBuffer(EWrapTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(DataPtr, SizeBytes)) {}
+
+ inline IoBuffer(ECloneTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(SizeBytes))
+ {
+ memcpy(const_cast<void*>(m_Core->DataPointer()), DataPtr, SizeBytes);
+ }
+
+ ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize);
+ ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize);
+
+ inline explicit operator bool() const { return !m_Core->IsNull(); }
+ inline operator MemoryView() const& { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); }
+ inline void MakeOwned() { return m_Core->MakeOwned(); }
+ [[nodiscard]] inline bool IsOwned() const { return m_Core->IsOwned(); }
+ [[nodiscard]] inline bool IsWholeFile() const { return m_Core->IsWholeFile(); }
+ [[nodiscard]] void* MutableData() const { return m_Core->MutableDataPointer(); }
+ void MakeImmutable() { m_Core->SetIsImmutable(true); }
+ [[nodiscard]] const void* Data() const { return m_Core->DataPointer(); }
+ [[nodiscard]] const void* GetData() const { return m_Core->DataPointer(); }
+ [[nodiscard]] size_t Size() const { return m_Core->DataBytes(); }
+ [[nodiscard]] size_t GetSize() const { return m_Core->DataBytes(); }
+ inline void SetContentType(ZenContentType ContentType) { m_Core->SetContentType(ContentType); }
+ [[nodiscard]] inline ZenContentType GetContentType() const { return m_Core->GetContentType(); }
+ [[nodiscard]] ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const;
+ void MarkAsDeleteOnClose();
+
+ inline MemoryView GetView() const { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); }
+ inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Core->MutableDataPointer(), m_Core->DataBytes()); }
+
+ template<typename T>
+ [[nodiscard]] const T* Data() const
+ {
+ return reinterpret_cast<const T*>(m_Core->DataPointer());
+ }
+
+ template<typename T>
+ [[nodiscard]] T* MutableData() const
+ {
+ return reinterpret_cast<T*>(m_Core->MutableDataPointer());
+ }
+
+private:
+ RefPtr<IoBufferCore> m_Core = new IoBufferCore;
+
+ IoBuffer(IoBufferCore* Core) : m_Core(Core) {}
+
+ friend class SharedBuffer;
+ friend class IoBufferBuilder;
+};
+
+class IoBufferBuilder
+{
+public:
+ ZENCORE_API static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
+ ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName);
+ ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull);
+ ZENCORE_API static IoBuffer ReadFromFileMaybe(IoBuffer& InBuffer);
+ inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
+ inline static IoBuffer MakeCloneFromMemory(MemoryView Memory) { return IoBuffer(IoBuffer::Clone, Memory.GetData(), Memory.GetSize()); }
+};
+
+IoHash HashBuffer(IoBuffer& Buffer);
+
+void iobuffer_forcelink();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h
new file mode 100644
index 000000000..fd0f4b2a7
--- /dev/null
+++ b/src/zencore/include/zencore/iohash.h
@@ -0,0 +1,115 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/blake3.h>
+#include <zencore/memory.h>
+
+#include <compare>
+#include <string_view>
+
+namespace zen {
+
+class StringBuilderBase;
+class CompositeBuffer;
+
+/**
+ * Hash used for content addressable storage
+ *
+ * This is basically a BLAKE3-160 hash (note: this is probably not an officially
+ * recognized identifier). It is generated by computing a 32-byte BLAKE3 hash and
+ * picking the first 20 bytes of the resulting hash.
+ *
+ */
+struct IoHash
+{
+ alignas(uint32_t) uint8_t Hash[20] = {};
+
+ static IoHash MakeFrom(const void* data /* 20 bytes */)
+ {
+ IoHash Io;
+ memcpy(Io.Hash, data, sizeof Io);
+ return Io;
+ }
+
+ static IoHash FromBLAKE3(const BLAKE3& Blake3)
+ {
+ IoHash Io;
+ memcpy(Io.Hash, Blake3.Hash, sizeof Io.Hash);
+ return Io;
+ }
+
+ static IoHash HashBuffer(const void* data, size_t byteCount);
+ static IoHash HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); }
+ static IoHash HashBuffer(const CompositeBuffer& Buffer);
+ static IoHash FromHexString(const char* string);
+ static IoHash FromHexString(const std::string_view string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+ std::string ToHexString() const;
+
+ static const int StringLength = 40;
+ typedef char String_t[StringLength + 1];
+
+ static const IoHash Zero; // Initialized to all zeros
+
+ inline auto operator<=>(const IoHash& rhs) const = default;
+
+ struct Hasher
+ {
+ size_t operator()(const IoHash& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct IoHashStream
+{
+ /// Begin streaming hash compute (not needed on freshly constructed instance)
+ void Reset() { m_Blake3Stream.Reset(); }
+
+ /// Append another chunk
+ IoHashStream& Append(const void* data, size_t byteCount)
+ {
+ m_Blake3Stream.Append(data, byteCount);
+ return *this;
+ }
+
+ /// Append another chunk
+ IoHashStream& Append(MemoryView Data)
+ {
+ m_Blake3Stream.Append(Data.GetData(), Data.GetSize());
+ return *this;
+ }
+
+ /// Obtain final hash. If you wish to reuse the instance call reset()
+ IoHash GetHash()
+ {
+ BLAKE3 b3 = m_Blake3Stream.GetHash();
+
+ IoHash Io;
+ memcpy(Io.Hash, b3.Hash, sizeof Io.Hash);
+
+ return Io;
+ }
+
+private:
+ BLAKE3Stream m_Blake3Stream;
+};
+
+} // namespace zen
+
+namespace std {
+
+template<>
+struct hash<zen::IoHash> : public zen::IoHash::Hasher
+{
+};
+
+} // namespace std
diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h
new file mode 100644
index 000000000..5cbe034cf
--- /dev/null
+++ b/src/zencore/include/zencore/logging.h
@@ -0,0 +1,136 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <spdlog/spdlog.h>
+#undef GetObject
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <string_view>
+
+namespace zen::logging {
+
+spdlog::logger& Default();
+void SetDefault(std::shared_ptr<spdlog::logger> NewDefaultLogger);
+spdlog::logger& ConsoleLog();
+spdlog::logger& Get(std::string_view Name);
+spdlog::logger* ErrorLog();
+void SetErrorLog(std::shared_ptr<spdlog::logger>&& NewErrorLogger);
+
+void InitializeLogging();
+void ShutdownLogging();
+
+} // namespace zen::logging
+
+namespace zen {
+extern spdlog::logger* TheDefaultLogger;
+
+inline spdlog::logger&
+Log()
+{
+ return *TheDefaultLogger;
+}
+
+using logging::ConsoleLog;
+using logging::ErrorLog;
+} // namespace zen
+
+using zen::ConsoleLog;
+using zen::ErrorLog;
+using zen::Log;
+
+struct LogCategory
+{
+ LogCategory(std::string_view InCategory) : Category(InCategory) {}
+
+ spdlog::logger& Logger()
+ {
+ static spdlog::logger& Inst = zen::logging::Get(Category);
+ return Inst;
+ }
+
+ std::string Category;
+};
+
+inline consteval bool
+LogIsErrorLevel(int level)
+{
+ return (level == spdlog::level::err || level == spdlog::level::critical);
+};
+
+#define ZEN_LOG_WITH_LOCATION(logger, loc, level, fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ if (logger.should_log(level)) \
+ { \
+ logger.log(loc, level, fmtstr, ##__VA_ARGS__); \
+ if (LogIsErrorLevel(level)) \
+ { \
+ if (auto ErrLogger = zen::logging::ErrorLog(); ErrLogger != nullptr) \
+ { \
+ ErrLogger->log(loc, level, fmtstr, ##__VA_ARGS__); \
+ } \
+ } \
+ } \
+ } while (false);
+
+#define ZEN_LOG(logger, level, fmtstr, ...) ZEN_LOG_WITH_LOCATION(logger, spdlog::source_loc{}, level, fmtstr, ##__VA_ARGS__)
+
+#define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \
+ static struct LogCategory##Category : public LogCategory \
+ { \
+ LogCategory##Category() : LogCategory(Name) {} \
+ } Category;
+
+#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::trace, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::debug, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::info, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::warn, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_LOG_ERROR(Category, fmtstr, ...) \
+ ZEN_LOG_WITH_LOCATION(Category.Logger(), \
+ spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \
+ spdlog::level::err, \
+ fmtstr##sv, \
+ ##__VA_ARGS__)
+
+#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) \
+ ZEN_LOG_WITH_LOCATION(Category.Logger(), \
+ spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \
+ spdlog::level::critical, \
+ fmtstr##sv, \
+ ##__VA_ARGS__)
+
+ // Helper macros for logging
+
+#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::trace, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::debug, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::info, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::warn, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_ERROR(fmtstr, ...) \
+ ZEN_LOG_WITH_LOCATION(Log(), spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), spdlog::level::err, fmtstr##sv, ##__VA_ARGS__)
+
+#define ZEN_CRITICAL(fmtstr, ...) \
+ ZEN_LOG_WITH_LOCATION(Log(), \
+ spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \
+ spdlog::level::critical, \
+ fmtstr##sv, \
+ ##__VA_ARGS__)
+
+#define ZEN_CONSOLE(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ ConsoleLog().info(fmtstr##sv, ##__VA_ARGS__); \
+ } while (false)
diff --git a/src/zencore/include/zencore/md5.h b/src/zencore/include/zencore/md5.h
new file mode 100644
index 000000000..d934dd86b
--- /dev/null
+++ b/src/zencore/include/zencore/md5.h
@@ -0,0 +1,50 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <stdint.h>
+#include <compare>
+#include "zencore.h"
+
+namespace zen {
+
+class StringBuilderBase;
+
+struct MD5
+{
+ uint8_t Hash[16];
+
+ inline auto operator<=>(const MD5& rhs) const = default;
+
+ static const int StringLength = 32;
+ typedef char String_t[StringLength + 1];
+
+ static MD5 HashMemory(const void* data, size_t byteCount);
+ static MD5 FromHexString(const char* string);
+ const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static MD5 Zero; // Initialized to all zeroes
+};
+
+/**
+ * Utility class for computing MD5 hashes
+ */
+class MD5Stream
+{
+public:
+ MD5Stream();
+
+ /// Begin streaming MD5 compute (not needed on freshly constructed MD5Stream instance)
+ void Reset();
+ /// Append another chunk
+ MD5Stream& Append(const void* data, size_t byteCount);
+ /// Obtain final MD5 hash. If you wish to reuse the MD5Stream instance call reset()
+ MD5 GetHash();
+
+private:
+};
+
+void md5_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/memory.h b/src/zencore/include/zencore/memory.h
new file mode 100644
index 000000000..560fa9ffc
--- /dev/null
+++ b/src/zencore/include/zencore/memory.h
@@ -0,0 +1,401 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/intmath.h>
+#include <zencore/thread.h>
+
+#include <cstddef>
+#include <cstring>
+#include <span>
+#include <vector>
+
+namespace zen {
+
+#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 201911L
+template<typename T>
+concept ContiguousRange = std::ranges::contiguous_range<T>;
+#else
+template<typename T>
+concept ContiguousRange = true;
+#endif
+
+struct MemoryView;
+
+class MemoryArena
+{
+public:
+ ZENCORE_API MemoryArena();
+ ZENCORE_API ~MemoryArena();
+
+ ZENCORE_API void* Alloc(size_t Size, size_t Alignment);
+ ZENCORE_API void Free(void* Ptr);
+
+private:
+};
+
+class Memory
+{
+public:
+ ZENCORE_API static void* Alloc(size_t Size, size_t Alignment = sizeof(void*));
+ ZENCORE_API static void Free(void* Ptr);
+};
+
+/** Allocator which claims fixed-size blocks from the underlying allocator.
+
+ There is no way to free individual memory blocks.
+
+ \note This is not thread-safe, you will need to provide synchronization yourself
+*/
+
+class ChunkingLinearAllocator
+{
+public:
+ ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment = sizeof(std::max_align_t));
+ ~ChunkingLinearAllocator();
+
+ ZENCORE_API void Reset();
+
+ ZENCORE_API void* Alloc(size_t Size, size_t Alignment = sizeof(void*));
+ inline void Free(void* Ptr) { ZEN_UNUSED(Ptr); /* no-op */ }
+
+ ChunkingLinearAllocator(const ChunkingLinearAllocator&) = delete;
+ ChunkingLinearAllocator& operator=(const ChunkingLinearAllocator&) = delete;
+
+private:
+ uint8_t* m_ChunkCursor = nullptr;
+ uint64_t m_ChunkBytesRemain = 0;
+ const uint64_t m_ChunkSize = 0;
+ const uint64_t m_ChunkAlignment = 0;
+ std::vector<void*> m_ChunkList;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct MutableMemoryView
+{
+ MutableMemoryView() = default;
+
+ MutableMemoryView(void* DataPtr, size_t DataSize)
+ : m_Data(reinterpret_cast<uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<uint8_t*>(DataPtr) + DataSize)
+ {
+ }
+
+ MutableMemoryView(void* DataPtr, void* DataEndPtr)
+ : m_Data(reinterpret_cast<uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<uint8_t*>(DataEndPtr))
+ {
+ }
+
+ inline bool IsEmpty() const { return m_Data == m_DataEnd; }
+ void* GetData() const { return m_Data; }
+ void* GetDataEnd() const { return m_DataEnd; }
+ size_t GetSize() const { return reinterpret_cast<uint8_t*>(m_DataEnd) - reinterpret_cast<uint8_t*>(m_Data); }
+
+ inline bool EqualBytes(const MutableMemoryView& InView) const
+ {
+ const size_t Size = GetSize();
+
+ return Size == InView.GetSize() && (memcmp(m_Data, InView.m_Data, Size) == 0);
+ }
+
+ /** Modifies the view to be the given number of bytes from the right. */
+ inline void RightInline(uint64_t InSize)
+ {
+ const uint64_t OldSize = GetSize();
+ const uint64_t NewSize = zen::Min(OldSize, InSize);
+ m_Data = GetDataAtOffsetNoCheck(OldSize - NewSize);
+ m_DataEnd = m_Data + NewSize;
+ }
+
+ /** Returns the right-most part of the view by taking the given number of bytes from the right. */
+ [[nodiscard]] inline MutableMemoryView Right(uint64_t InSize) const
+ {
+ MutableMemoryView View(*this);
+ View.RightChopInline(InSize);
+ return View;
+ }
+
+ /** Modifies the view by chopping the given number of bytes from the left. */
+ inline void RightChopInline(uint64_t InSize)
+ {
+ const uint64_t Offset = zen::Min(GetSize(), InSize);
+ m_Data = GetDataAtOffsetNoCheck(Offset);
+ }
+
+ /** Returns the left-most part of the view by taking the given number of bytes from the left. */
+ constexpr inline MutableMemoryView Left(uint64_t InSize) const
+ {
+ MutableMemoryView View(*this);
+ View.LeftInline(InSize);
+ return View;
+ }
+
+ /** Modifies the view to be the given number of bytes from the left. */
+ constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = zen::Min(m_DataEnd, m_Data + InSize); }
+
+ /** Modifies the view to be the middle part by taking up to the given number of bytes from the given offset. */
+ inline void MidInline(uint64_t InOffset, uint64_t InSize = ~uint64_t(0))
+ {
+ RightChopInline(InOffset);
+ LeftInline(InSize);
+ }
+
+ /** Returns the middle part of the view by taking up to the given number of bytes from the given position. */
+ [[nodiscard]] inline MutableMemoryView Mid(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) const
+ {
+ MutableMemoryView View(*this);
+ View.MidInline(InOffset, InSize);
+ return View;
+ }
+
+ /** Returns the right-most part of the view by chopping the given number of bytes from the left. */
+ [[nodiscard]] inline MutableMemoryView RightChop(uint64_t InSize) const
+ {
+ MutableMemoryView View(*this);
+ View.RightChopInline(InSize);
+ return View;
+ }
+
+ inline MutableMemoryView& operator+=(size_t InSize)
+ {
+ RightChopInline(InSize);
+ return *this;
+ }
+
+ /** Copies bytes from the input view into this view, and returns the remainder of this view. */
+ inline MutableMemoryView CopyFrom(MemoryView InView) const;
+
+private:
+ uint8_t* m_Data = nullptr;
+ uint8_t* m_DataEnd = nullptr;
+
+ /** Returns the data pointer advanced by an offset in bytes. */
+ inline constexpr uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct MemoryView
+{
+ MemoryView() = default;
+
+ MemoryView(const MutableMemoryView& MutableView)
+ : m_Data(reinterpret_cast<const uint8_t*>(MutableView.GetData()))
+ , m_DataEnd(m_Data + MutableView.GetSize())
+ {
+ }
+
+ MemoryView(const void* DataPtr, size_t DataSize)
+ : m_Data(reinterpret_cast<const uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<const uint8_t*>(DataPtr) + DataSize)
+ {
+ }
+
+ MemoryView(const void* DataPtr, const void* DataEndPtr)
+ : m_Data(reinterpret_cast<const uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<const uint8_t*>(DataEndPtr))
+ {
+ }
+
+ inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); }
+ inline bool IsEmpty() const { return m_Data == m_DataEnd; }
+ const void* GetData() const { return m_Data; }
+ const void* GetDataEnd() const { return m_DataEnd; }
+ size_t GetSize() const { return reinterpret_cast<const uint8_t*>(m_DataEnd) - reinterpret_cast<const uint8_t*>(m_Data); }
+ inline bool operator==(const MemoryView& Rhs) const { return m_Data == Rhs.m_Data && m_DataEnd == Rhs.m_DataEnd; }
+
+ inline bool EqualBytes(const MemoryView& InView) const
+ {
+ const size_t Size = GetSize();
+
+ return Size == InView.GetSize() && (memcmp(m_Data, InView.GetData(), Size) == 0);
+ }
+
+ inline MemoryView& operator+=(size_t InSize)
+ {
+ RightChopInline(InSize);
+ return *this;
+ }
+
+ /** Modifies the view by chopping the given number of bytes from the left. */
+ inline void RightChopInline(uint64_t InSize)
+ {
+ const uint64_t Offset = zen::Min(GetSize(), InSize);
+ m_Data = GetDataAtOffsetNoCheck(Offset);
+ }
+
+ inline MemoryView RightChop(uint64_t InSize)
+ {
+ MemoryView View(*this);
+ View.RightChopInline(InSize);
+ return View;
+ }
+
+ /** Returns the right-most part of the view by taking the given number of bytes from the right. */
+ [[nodiscard]] inline MemoryView Right(uint64_t InSize) const
+ {
+ MemoryView View(*this);
+ View.RightInline(InSize);
+ return View;
+ }
+
+ /** Modifies the view to be the given number of bytes from the right. */
+ inline void RightInline(uint64_t InSize)
+ {
+ const uint64_t OldSize = GetSize();
+ const uint64_t NewSize = zen::Min(OldSize, InSize);
+ m_Data = GetDataAtOffsetNoCheck(OldSize - NewSize);
+ m_DataEnd = m_Data + NewSize;
+ }
+
+ /** Returns the left-most part of the view by taking the given number of bytes from the left. */
+ inline MemoryView Left(uint64_t InSize) const
+ {
+ MemoryView View(*this);
+ View.LeftInline(InSize);
+ return View;
+ }
+
+ /** Modifies the view to be the given number of bytes from the left. */
+ inline void LeftInline(uint64_t InSize)
+ {
+ InSize = zen::Min(GetSize(), InSize);
+ m_DataEnd = zen::Min(m_DataEnd, m_Data + InSize);
+ }
+
+ /** Modifies the view to be the middle part by taking up to the given number of bytes from the given offset. */
+ inline void MidInline(uint64_t InOffset, uint64_t InSize = ~uint64_t(0))
+ {
+ RightChopInline(InOffset);
+ LeftInline(InSize);
+ }
+
+ /** Returns the middle part of the view by taking up to the given number of bytes from the given position. */
+ [[nodiscard]] inline MemoryView Mid(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) const
+ {
+ MemoryView View(*this);
+ View.MidInline(InOffset, InSize);
+ return View;
+ }
+
+ constexpr void Reset()
+ {
+ m_Data = nullptr;
+ m_DataEnd = nullptr;
+ }
+
+private:
+ const uint8_t* m_Data = nullptr;
+ const uint8_t* m_DataEnd = nullptr;
+
+ /** Returns the data pointer advanced by an offset in bytes. */
+ inline constexpr const uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; }
+};
+
+inline MutableMemoryView
+MutableMemoryView::CopyFrom(MemoryView InView) const
+{
+ ZEN_ASSERT(InView.GetSize() <= GetSize());
+ memcpy(m_Data, InView.GetData(), InView.GetSize());
+ return RightChop(InView.GetSize());
+}
+
+/** Advances the start of the view by an offset, which is clamped to stay within the view. */
+inline MemoryView
+operator+(const MemoryView& View, uint64_t Offset)
+{
+ return MemoryView(View) += Offset;
+}
+
+/** Advances the start of the view by an offset, which is clamped to stay within the view. */
+inline MemoryView
+operator+(uint64_t Offset, const MemoryView& View)
+{
+ return MemoryView(View) += Offset;
+}
+
+/** Advances the start of the view by an offset, which is clamped to stay within the view. */
+inline MutableMemoryView
+operator+(const MutableMemoryView& View, uint64_t Offset)
+{
+ return MutableMemoryView(View) += Offset;
+}
+
+/** Advances the start of the view by an offset, which is clamped to stay within the view. */
+inline MutableMemoryView
+operator+(uint64_t Offset, const MutableMemoryView& View)
+{
+ return MutableMemoryView(View) += Offset;
+}
+
+/**
+ * Make a non-owning view of the memory of the initializer list.
+ *
+ * This overload is only available when the element type does not need to be deduced.
+ */
+template<typename T>
+[[nodiscard]] inline MemoryView
+MakeMemoryView(std::initializer_list<typename std::type_identity<T>::type> List)
+{
+ return MemoryView(List.begin(), List.size() * sizeof(T));
+}
+
+/** Make a non-owning view of the memory of the contiguous container. */
+template<ContiguousRange R>
+[[nodiscard]] constexpr inline MemoryView
+MakeMemoryView(const R& Container)
+{
+ std::span Span = Container;
+ return MemoryView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type));
+}
+
+/** Make a non-owning const view starting at Data and ending at DataEnd. */
+
+[[nodiscard]] inline MemoryView
+MakeMemoryView(const void* Data, const void* DataEnd)
+{
+ return MemoryView(Data, DataEnd);
+}
+
+[[nodiscard]] inline MemoryView
+MakeMemoryView(const void* Data, uint64_t Size)
+{
+ return MemoryView(Data, reinterpret_cast<const uint8_t*>(Data) + Size);
+}
+
+/**
+ * Make a non-owning mutable view of the memory of the initializer list.
+ *
+ * This overload is only available when the element type does not need to be deduced.
+ */
+template<typename T>
+[[nodiscard]] inline MutableMemoryView
+MakeMutableMemoryView(std::initializer_list<typename std::type_identity<T>::type> List)
+{
+ return MutableMemoryView(List.begin(), List.size() * sizeof(T));
+}
+
+/** Make a non-owning mutable view of the memory of the contiguous container. */
+template<ContiguousRange R>
+[[nodiscard]] constexpr inline MutableMemoryView
+MakeMutableMemoryView(R& Container)
+{
+ std::span Span = Container;
+ return MutableMemoryView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type));
+}
+
+/** Make a non-owning mutable view starting at Data and ending at DataEnd. */
+
+[[nodiscard]] inline MutableMemoryView
+MakeMutableMemoryView(void* Data, void* DataEnd)
+{
+ return MutableMemoryView(Data, DataEnd);
+}
+
+void memory_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/meta.h b/src/zencore/include/zencore/meta.h
new file mode 100644
index 000000000..82eb5cc30
--- /dev/null
+++ b/src/zencore/include/zencore/meta.h
@@ -0,0 +1,30 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/* This file contains utility functions for meta programming
+ *
+ * Since you're in here you're probably quite observant, and you'll
+ * note that it's quite barren here. This is because template
+ * metaprogramming is awful and I try not to engage in it. However,
+ * sometimes these things are forced upon us.
+ *
+ */
+
+namespace zen {
+
+/**
+ * Uses implicit conversion to create an instance of a specific type.
+ * Useful to make things clearer or circumvent unintended type deduction in templates.
+ * Safer than C casts and static_casts, e.g. does not allow down-casts
+ *
+ * @param Obj The object (usually pointer or reference) to convert.
+ *
+ * @return The object converted to the specified type.
+ */
+template<typename T>
+inline T
+ImplicitConv(typename std::type_identity<T>::type Obj)
+{
+ return Obj;
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/mpscqueue.h b/src/zencore/include/zencore/mpscqueue.h
new file mode 100644
index 000000000..19e410d85
--- /dev/null
+++ b/src/zencore/include/zencore/mpscqueue.h
@@ -0,0 +1,110 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <atomic>
+#include <memory>
+#include <new>
+#include <optional>
+
+#ifdef __cpp_lib_hardware_interference_size
+using std::hardware_constructive_interference_size;
+using std::hardware_destructive_interference_size;
+#else
+// 64 bytes on x86-64 │ L1_CACHE_BYTES │ L1_CACHE_SHIFT │ __cacheline_aligned │ ...
+constexpr std::size_t hardware_constructive_interference_size = 64;
+constexpr std::size_t hardware_destructive_interference_size = 64;
+#endif
+
+namespace zen {
+
+/** An untyped array of data with compile-time alignment and size derived from another type. */
+template<typename ElementType>
+struct TypeCompatibleStorage
+{
+ ElementType* Data() { return (ElementType*)this; }
+ const ElementType* Data() const { return (const ElementType*)this; }
+
+ alignas(ElementType) char DataMember;
+};
+
+/** Fast multi-producer/single-consumer unbounded concurrent queue.
+
+ Based on http://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue
+ */
+
+template<typename T>
+class MpscQueue final
+{
+public:
+ using ElementType = T;
+
+ MpscQueue()
+ {
+ Node* Sentinel = new Node;
+ Head.store(Sentinel, std::memory_order_relaxed);
+ Tail = Sentinel;
+ }
+
+ ~MpscQueue()
+ {
+ Node* Next = Tail->Next.load(std::memory_order_relaxed);
+
+ // sentinel's value is already destroyed
+ delete Tail;
+
+ while (Next != nullptr)
+ {
+ Tail = Next;
+ Next = Tail->Next.load(std::memory_order_relaxed);
+
+ std::destroy_at((ElementType*)&Tail->Value);
+ delete Tail;
+ }
+ }
+
+ template<typename... ArgTypes>
+ void Enqueue(ArgTypes&&... Args)
+ {
+ Node* New = new Node;
+ new (&New->Value) ElementType(std::forward<ArgTypes>(Args)...);
+
+ Node* Prev = Head.exchange(New, std::memory_order_acq_rel);
+ Prev->Next.store(New, std::memory_order_release);
+ }
+
+ std::optional<ElementType> Dequeue()
+ {
+ Node* Next = Tail->Next.load(std::memory_order_acquire);
+
+ if (Next == nullptr)
+ {
+ return {};
+ }
+
+ ElementType* ValuePtr = (ElementType*)&Next->Value;
+ std::optional<ElementType> Res{std::move(*ValuePtr)};
+ std::destroy_at(ValuePtr);
+
+ delete Tail; // current sentinel
+
+ Tail = Next; // new sentinel
+ return Res;
+ }
+
+private:
+ struct Node
+ {
+ std::atomic<Node*> Next{nullptr};
+ TypeCompatibleStorage<ElementType> Value;
+ };
+
+private:
+ std::atomic<Node*> Head; // accessed only by producers
+ alignas(hardware_constructive_interference_size)
+ Node* Tail; // accessed only by consumer, hence should be on a different cache line than `Head`
+};
+
+void mpscqueue_forcelink();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/refcount.h b/src/zencore/include/zencore/refcount.h
new file mode 100644
index 000000000..f0bb6b85e
--- /dev/null
+++ b/src/zencore/include/zencore/refcount.h
@@ -0,0 +1,186 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+#pragma once
+
+#include "atomic.h"
+#include "zencore.h"
+
+#include <compare>
+
+namespace zen {
+
+/**
+ * Helper base class for reference counted objects using intrusive reference counts
+ *
+ * This class is pretty straightforward but does one thing which may be unexpected:
+ *
+ * - Instances on the stack are initialized with a reference count of one to ensure
+ * nobody tries to accidentally delete it. (TODO: is this really useful?)
+ */
+class RefCounted
+{
+public:
+ RefCounted() = default;
+ virtual ~RefCounted() = default;
+
+ inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); }
+ inline uint32_t Release() const
+ {
+ uint32_t refCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount);
+ if (refCount == 0)
+ {
+ delete this;
+ }
+ return refCount;
+ }
+
+ // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it
+
+ RefCounted(const RefCounted&) = delete;
+ RefCounted(RefCounted&&) = delete;
+ RefCounted& operator=(const RefCounted&) = delete;
+ RefCounted& operator=(RefCounted&&) = delete;
+
+protected:
+ inline uint32_t RefCount() const { return m_RefCount; }
+
+private:
+ uint32_t m_RefCount = 0;
+};
+
+/**
+ * Smart pointer for classes derived from RefCounted
+ */
+
+template<class T>
+class RefPtr
+{
+public:
+ inline RefPtr() = default;
+ inline RefPtr(const RefPtr& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); }
+ inline RefPtr(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); }
+ inline ~RefPtr() { m_Ref && m_Ref->Release(); }
+
+ [[nodiscard]] inline bool IsNull() const { return m_Ref == nullptr; }
+ inline explicit operator bool() const { return m_Ref != nullptr; }
+ inline operator T*() const { return m_Ref; }
+ inline T* operator->() const { return m_Ref; }
+
+ inline std::strong_ordering operator<=>(const RefPtr& Rhs) const = default;
+
+ inline RefPtr& operator=(T* Rhs)
+ {
+ Rhs && Rhs->AddRef();
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs;
+ return *this;
+ }
+ inline RefPtr& operator=(const RefPtr& Rhs)
+ {
+ if (&Rhs != this)
+ {
+ Rhs && Rhs->AddRef();
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ }
+ return *this;
+ }
+ inline RefPtr& operator=(RefPtr&& Rhs) noexcept
+ {
+ if (&Rhs != this)
+ {
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ Rhs.m_Ref = nullptr;
+ }
+ return *this;
+ }
+ template<typename OtherType>
+ inline RefPtr& operator=(RefPtr<OtherType>&& Rhs) noexcept
+ {
+ if ((RefPtr*)&Rhs != this)
+ {
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ Rhs.m_Ref = nullptr;
+ }
+ return *this;
+ }
+ inline RefPtr(RefPtr&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
+ template<typename OtherType>
+ explicit inline RefPtr(RefPtr<OtherType>&& Rhs) noexcept : m_Ref(Rhs.m_Ref)
+ {
+ Rhs.m_Ref = nullptr;
+ }
+
+private:
+ T* m_Ref = nullptr;
+ template<typename U>
+ friend class RefPtr;
+};
+
+/**
+ * Smart pointer for classes derived from RefCounted
+ *
+ * This variant does not decay to a raw pointer
+ *
+ */
+
+template<class T>
+class Ref
+{
+public:
+ inline Ref() = default;
+ inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); }
+ inline explicit Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); }
+ inline ~Ref() { m_Ref && m_Ref->Release(); }
+
+ template<typename DerivedType>
+ requires DerivedFrom<DerivedType, T>
+ inline Ref(const Ref<DerivedType>& Rhs) : Ref(Rhs.m_Ref) {}
+
+ [[nodiscard]] inline bool IsNull() const { return m_Ref == nullptr; }
+ inline explicit operator bool() const { return m_Ref != nullptr; }
+ inline T* operator->() const { return m_Ref; }
+ inline T* Get() const { return m_Ref; }
+
+ inline std::strong_ordering operator<=>(const Ref& Rhs) const = default;
+
+ inline Ref& operator=(T* Rhs)
+ {
+ Rhs && Rhs->AddRef();
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs;
+ return *this;
+ }
+ inline Ref& operator=(const Ref& Rhs)
+ {
+ if (&Rhs != this)
+ {
+ Rhs && Rhs->AddRef();
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ }
+ return *this;
+ }
+ inline Ref& operator=(Ref&& Rhs) noexcept
+ {
+ if (&Rhs != this)
+ {
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ Rhs.m_Ref = nullptr;
+ }
+ return *this;
+ }
+ inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
+
+private:
+ T* m_Ref = nullptr;
+
+ template<class U>
+ friend class Ref;
+};
+
+void refcount_forcelink();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/scopeguard.h b/src/zencore/include/zencore/scopeguard.h
new file mode 100644
index 000000000..d04c8ed9c
--- /dev/null
+++ b/src/zencore/include/zencore/scopeguard.h
@@ -0,0 +1,45 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <type_traits>
+#include "logging.h"
+#include "zencore.h"
+
+namespace zen {
+
+template<typename T>
+class [[nodiscard]] ScopeGuardImpl
+{
+public:
+ inline ScopeGuardImpl(T&& func) : m_guardFunc(func) {}
+ ~ScopeGuardImpl()
+ {
+ if (!m_dismissed)
+ {
+ try
+ {
+ m_guardFunc();
+ }
+ catch (std::exception& Ex)
+ {
+ ZEN_ERROR("scope guard threw exception: '{}'", Ex.what());
+ }
+ }
+ }
+
+ void Dismiss() { m_dismissed = true; }
+
+private:
+ bool m_dismissed = false;
+ T m_guardFunc;
+};
+
+template<typename T>
+ScopeGuardImpl<T>
+MakeGuard(T&& fn)
+{
+ return ScopeGuardImpl<T>(std::move(fn));
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/session.h b/src/zencore/include/zencore/session.h
new file mode 100644
index 000000000..dd90197bf
--- /dev/null
+++ b/src/zencore/include/zencore/session.h
@@ -0,0 +1,14 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+namespace zen {
+
+struct Oid;
+
+ZENCORE_API [[nodiscard]] Oid GetSessionId();
+ZENCORE_API [[nodiscard]] std::string_view GetSessionIdString();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/sha1.h b/src/zencore/include/zencore/sha1.h
new file mode 100644
index 000000000..fc26f442b
--- /dev/null
+++ b/src/zencore/include/zencore/sha1.h
@@ -0,0 +1,76 @@
+// //////////////////////////////////////////////////////////
+// sha1.h
+// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved.
+// see http://create.stephan-brumme.com/disclaimer.html
+//
+
+#pragma once
+
+#include <stdint.h>
+#include <compare>
+#include "zencore.h"
+
+namespace zen {
+
+class StringBuilderBase;
+
+struct SHA1
+{
+ uint8_t Hash[20];
+
+ inline auto operator<=>(const SHA1& rhs) const = default;
+
+ static const int StringLength = 40;
+ typedef char String_t[StringLength + 1];
+
+ static SHA1 HashMemory(const void* data, size_t byteCount);
+ static SHA1 FromHexString(const char* string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static SHA1 Zero; // Initialized to all zeroes
+};
+
+/**
+ * Utility class for computing SHA1 hashes
+ */
+class SHA1Stream
+{
+public:
+ SHA1Stream();
+
+ /** compute SHA1 of a memory block
+
+ \note SHA1 class contains a slightly more convenient helper function for this use case
+ \see SHA1::fromMemory()
+ */
+ SHA1 Compute(const void* data, size_t byteCount);
+
+ /// Begin streaming SHA1 compute (not needed on freshly constructed SHA1Stream instance)
+ void Reset();
+ /// Append another chunk
+ SHA1Stream& Append(const void* data, size_t byteCount);
+ /// Obtain final SHA1 hash. If you wish to reuse the SHA1Stream instance call reset()
+ SHA1 GetHash();
+
+private:
+ void ProcessBlock(const void* data);
+ void ProcessBuffer();
+
+ enum
+ {
+ /// split into 64 byte blocks (=> 512 bits)
+ BlockSize = 512 / 8,
+ HashBytes = 20,
+ HashValues = HashBytes / 4
+ };
+
+ uint64_t m_NumBytes; // size of processed data in bytes
+ size_t m_BufferSize; // valid bytes in m_buffer
+ uint8_t m_Buffer[BlockSize]; // bytes not processed yet
+ uint32_t m_Hash[HashValues];
+};
+
+void sha1_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/sharedbuffer.h b/src/zencore/include/zencore/sharedbuffer.h
new file mode 100644
index 000000000..97c5a9d21
--- /dev/null
+++ b/src/zencore/include/zencore/sharedbuffer.h
@@ -0,0 +1,167 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/iobuffer.h>
+#include <zencore/memory.h>
+#include <zencore/refcount.h>
+
+#include <memory.h>
+
+namespace zen {
+
+class SharedBuffer;
+
+/**
+ * Reference to a memory buffer with a single owner
+ *
+ * Internally
+ */
+class UniqueBuffer
+{
+public:
+ UniqueBuffer() = default;
+ UniqueBuffer(UniqueBuffer&&) = default;
+ UniqueBuffer& operator=(UniqueBuffer&&) = default;
+ UniqueBuffer(const UniqueBuffer&) = delete;
+ UniqueBuffer& operator=(const UniqueBuffer&) = delete;
+
+ ZENCORE_API explicit UniqueBuffer(IoBufferCore* Owner);
+
+ [[nodiscard]] void* GetData() { return m_Buffer ? m_Buffer->MutableDataPointer() : nullptr; }
+ [[nodiscard]] const void* GetData() const { return m_Buffer ? m_Buffer->DataPointer() : nullptr; }
+ [[nodiscard]] size_t GetSize() const { return m_Buffer ? m_Buffer->DataBytes() : 0; }
+
+ operator MutableMemoryView() { return GetMutableView(); }
+ operator MemoryView() const { return GetView(); }
+
+ /**
+ * Returns true if this does not point to a buffer owner.
+ *
+ * A null buffer is always owned, materialized, and empty.
+ */
+ [[nodiscard]] inline bool IsNull() const { return m_Buffer.IsNull(); }
+
+ /** Reset this to null. */
+ ZENCORE_API void Reset();
+
+ [[nodiscard]] inline MutableMemoryView GetMutableView() { return MutableMemoryView(GetData(), GetSize()); }
+ [[nodiscard]] inline MemoryView GetView() const { return MemoryView(GetData(), GetSize()); }
+
+ /** Make an uninitialized owned buffer of the specified size. */
+ [[nodiscard]] ZENCORE_API static UniqueBuffer Alloc(uint64_t Size);
+
+ /** Make a non-owned view of the input. */
+ [[nodiscard]] ZENCORE_API static UniqueBuffer MakeMutableView(void* DataPtr, uint64_t Size);
+
+ /**
+ * Convert this to an immutable shared buffer, leaving this null.
+ *
+ * Steals the buffer owner from the unique buffer.
+ */
+ [[nodiscard]] ZENCORE_API SharedBuffer MoveToShared();
+
+private:
+ // This may be null, for a default constructed UniqueBuffer only
+ RefPtr<IoBufferCore> m_Buffer;
+
+ friend class SharedBuffer;
+};
+
+/**
+ * Reference to a memory buffer with shared ownership
+ */
+class SharedBuffer
+{
+public:
+ SharedBuffer() = default;
+ ZENCORE_API explicit SharedBuffer(UniqueBuffer&&);
+ inline explicit SharedBuffer(IoBufferCore* Owner) : m_Buffer(Owner) {}
+ ZENCORE_API explicit SharedBuffer(IoBuffer&& Buffer) : m_Buffer(std::move(Buffer.m_Core)) {}
+ ZENCORE_API explicit SharedBuffer(const IoBuffer& Buffer) : m_Buffer(Buffer.m_Core) {}
+ ZENCORE_API explicit SharedBuffer(RefPtr<IoBufferCore>&& Owner) : m_Buffer(std::move(Owner)) {}
+
+ [[nodiscard]] const void* GetData() const
+ {
+ if (m_Buffer)
+ {
+ return m_Buffer->DataPointer();
+ }
+ return nullptr;
+ }
+
+ [[nodiscard]] size_t GetSize() const
+ {
+ if (m_Buffer)
+ {
+ return m_Buffer->DataBytes();
+ }
+ return 0;
+ }
+
+ inline void MakeImmutable()
+ {
+ ZEN_ASSERT(m_Buffer);
+ m_Buffer->SetIsImmutable(true);
+ }
+
+ /** Returns a buffer that is owned, by cloning if not owned. */
+ [[nodiscard]] ZENCORE_API SharedBuffer MakeOwned() const&;
+ [[nodiscard]] ZENCORE_API SharedBuffer MakeOwned() &&;
+
+ [[nodiscard]] bool IsOwned() const { return !m_Buffer || m_Buffer->IsOwned(); }
+ [[nodiscard]] inline bool IsNull() const { return !m_Buffer; }
+ inline void Reset() { m_Buffer = nullptr; }
+
+ [[nodiscard]] MemoryView GetView() const
+ {
+ if (m_Buffer)
+ {
+ return MemoryView(m_Buffer->DataPointer(), m_Buffer->DataBytes());
+ }
+ else
+ {
+ return MemoryView();
+ }
+ }
+ operator MemoryView() const { return GetView(); }
+
+ /** Returns true if this points to a buffer owner. */
+ [[nodiscard]] inline explicit operator bool() const { return !IsNull(); }
+
+ [[nodiscard]] inline IoBuffer AsIoBuffer() const { return IoBuffer(m_Buffer); }
+
+ SharedBuffer& operator=(UniqueBuffer&& Rhs)
+ {
+ m_Buffer = std::move(Rhs.m_Buffer);
+ return *this;
+ }
+
+ std::strong_ordering operator<=>(const SharedBuffer& Rhs) const = default;
+
+ /** Make a non-owned view of the input */
+ [[nodiscard]] inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); }
+ /** Make a non-owning view of the memory of the contiguous container. */
+ [[nodiscard]] inline static SharedBuffer MakeView(const ContiguousRange auto& Container)
+ {
+ std::span Span = Container;
+ return MakeView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type));
+ }
+ /** Make a non-owned view of the input */
+ [[nodiscard]] ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size);
+ /** Make a non-owned view of the input */
+ [[nodiscard]] ZENCORE_API static SharedBuffer MakeView(MemoryView View, SharedBuffer OuterBuffer);
+ /** Make an owned clone of the buffer */
+ [[nodiscard]] ZENCORE_API SharedBuffer Clone();
+ /** Make an owned clone of the memory in the input view */
+ [[nodiscard]] ZENCORE_API static SharedBuffer Clone(MemoryView View);
+
+private:
+ RefPtr<IoBufferCore> m_Buffer;
+};
+
+void sharedbuffer_forcelink();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/stats.h b/src/zencore/include/zencore/stats.h
new file mode 100644
index 000000000..1a0817b99
--- /dev/null
+++ b/src/zencore/include/zencore/stats.h
@@ -0,0 +1,295 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <atomic>
+#include <random>
+
+namespace zen {
+class CbObjectWriter;
+}
+
+namespace zen::metrics {
+
+template<typename T>
+class Gauge
+{
+public:
+ Gauge() : m_Value{0} {}
+
+ T Value() const { return m_Value; }
+ void SetValue(T Value) { m_Value = Value; }
+
+private:
+ std::atomic<T> m_Value;
+};
+
+/** Stats counter
+ *
+ * A counter is modified by adding or subtracting a value from a current value.
+ * This would typically be used to track number of requests in flight, number
+ * of active jobs etc
+ *
+ */
+class Counter
+{
+public:
+ inline void SetValue(uint64_t Value) { m_count = Value; }
+ inline uint64_t Value() const { return m_count; }
+
+ inline void Increment(int64_t AddValue) { m_count.fetch_add(AddValue); }
+ inline void Decrement(int64_t SubValue) { m_count.fetch_sub(SubValue); }
+ inline void Clear() { m_count.store(0, std::memory_order_release); }
+
+private:
+ std::atomic<uint64_t> m_count{0};
+};
+
+/** Exponential Weighted Moving Average
+
+ This is very raw, to use as little state as possible. If we
+ want to use this more broadly in user code we should perhaps
+ add a more user-friendly wrapper
+ */
+
+class RawEWMA
+{
+public:
+ /// <summary>
+ /// Update EWMA with new measure
+ /// </summary>
+ /// <param name="Alpha">Smoothing factor (between 0 and 1)</param>
+ /// <param name="Interval">Elapsed time since last</param>
+ /// <param name="Count">Value</param>
+ /// <param name="IsInitialUpdate">Whether this is the first update or not</param>
+ void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate);
+ double Rate() const;
+
+private:
+ std::atomic<double> m_Rate = 0;
+};
+
+/// <summary>
+/// Tracks rate of events over time (i.e requests/sec), using
+/// exponential moving averages
+/// </summary>
+class Meter
+{
+public:
+ Meter();
+ ~Meter();
+
+ inline uint64_t Count() const { return m_TotalCount; }
+ double Rate1(); // One-minute rate
+ double Rate5(); // Five-minute rate
+ double Rate15(); // Fifteen-minute rate
+ double MeanRate() const; // Mean rate since instantiation of this meter
+ void Mark(uint64_t Count = 1); // Register one or more events
+
+private:
+ std::atomic<uint64_t> m_TotalCount{0}; // Accumulator counting number of marks since beginning
+ std::atomic<uint64_t> m_PendingCount{0}; // Pending EWMA update accumulator
+ std::atomic<uint64_t> m_StartTick{0}; // Time this was instantiated (for mean)
+ std::atomic<uint64_t> m_LastTick{0}; // Timestamp of last EWMA tick
+ std::atomic<int64_t> m_Remainder{0}; // Tracks the "modulo" of tick time
+ bool m_IsFirstTick = true;
+ RawEWMA m_RateM1;
+ RawEWMA m_RateM5;
+ RawEWMA m_RateM15;
+
+ void TickIfNecessary();
+ void Tick();
+};
+
+/** Moment-in-time snapshot of a distribution
+ */
+class SampleSnapshot
+{
+public:
+ SampleSnapshot(std::vector<double>&& Values);
+ ~SampleSnapshot();
+
+ uint32_t Size() const { return (uint32_t)m_Values.size(); }
+ double GetQuantileValue(double Quantile);
+ double GetMedian() { return GetQuantileValue(0.5); }
+ double Get75Percentile() { return GetQuantileValue(0.75); }
+ double Get95Percentile() { return GetQuantileValue(0.95); }
+ double Get98Percentile() { return GetQuantileValue(0.98); }
+ double Get99Percentile() { return GetQuantileValue(0.99); }
+ double Get999Percentile() { return GetQuantileValue(0.999); }
+ const std::vector<double>& GetValues() const;
+
+private:
+ std::vector<double> m_Values;
+};
+
+/** Randomly selects samples from a stream. Uses Vitter's
+ Algorithm R to produce a statistically representative sample.
+
+ http://www.cs.umd.edu/~samir/498/vitter.pdf - Random Sampling with a Reservoir
+ */
+
+class UniformSample
+{
+public:
+ UniformSample(uint32_t ReservoirSize);
+ ~UniformSample();
+
+ void Clear();
+ uint32_t Size() const;
+ void Update(int64_t Value);
+ SampleSnapshot Snapshot() const;
+
+ template<Invocable<int64_t> T>
+ void IterateValues(T Callback) const
+ {
+ for (const auto& Value : m_Values)
+ {
+ Callback(Value);
+ }
+ }
+
+private:
+ std::atomic<uint64_t> m_SampleCounter{0};
+ std::vector<std::atomic<int64_t>> m_Values;
+};
+
+/** Track (probabilistic) sample distribution along with min/max
+ */
+class Histogram
+{
+public:
+ Histogram(int32_t SampleCount = 1028);
+ ~Histogram();
+
+ void Clear();
+ void Update(int64_t Value);
+ int64_t Max() const;
+ int64_t Min() const;
+ double Mean() const;
+ uint64_t Count() const;
+ SampleSnapshot Snapshot() const { return m_Sample.Snapshot(); }
+
+private:
+ UniformSample m_Sample;
+ std::atomic<int64_t> m_Min{0};
+ std::atomic<int64_t> m_Max{0};
+ std::atomic<int64_t> m_Sum{0};
+ std::atomic<int64_t> m_Count{0};
+};
+
+/** Track timing and frequency of some operation
+
+ Example usage would be to track frequency and duration of network
+ requests, or function calls.
+
+ */
+class OperationTiming
+{
+public:
+ OperationTiming(int32_t SampleCount = 514);
+ ~OperationTiming();
+
+ void Update(int64_t Duration);
+ int64_t Max() const;
+ int64_t Min() const;
+ double Mean() const;
+ uint64_t Count() const;
+ SampleSnapshot Snapshot() const { return m_Histogram.Snapshot(); }
+
+ double Rate1() { return m_Meter.Rate1(); }
+ double Rate5() { return m_Meter.Rate5(); }
+ double Rate15() { return m_Meter.Rate15(); }
+ double MeanRate() const { return m_Meter.MeanRate(); }
+
+ struct Scope
+ {
+ Scope(OperationTiming& Outer);
+ ~Scope();
+
+ void Stop();
+ void Cancel();
+
+ private:
+ OperationTiming& m_Outer;
+ uint64_t m_StartTick;
+ };
+
+private:
+ Meter m_Meter;
+ Histogram m_Histogram;
+};
+
+/** Metrics for network requests
+
+ Aggregates tracking of duration, payload sizes into a single
+ class
+
+ */
+class RequestStats
+{
+public:
+ RequestStats(int32_t SampleCount = 514);
+ ~RequestStats();
+
+ void Update(int64_t Duration, int64_t Bytes);
+ uint64_t Count() const;
+
+ // Timing
+
+ int64_t MaxDuration() const { return m_BytesHistogram.Max(); }
+ int64_t MinDuration() const { return m_BytesHistogram.Min(); }
+ double MeanDuration() const { return m_BytesHistogram.Mean(); }
+ SampleSnapshot DurationSnapshot() const { return m_RequestTimeHistogram.Snapshot(); }
+ double Rate1() { return m_RequestMeter.Rate1(); }
+ double Rate5() { return m_RequestMeter.Rate5(); }
+ double Rate15() { return m_RequestMeter.Rate15(); }
+ double MeanRate() const { return m_RequestMeter.MeanRate(); }
+
+ // Bytes
+
+ int64_t MaxBytes() const { return m_BytesHistogram.Max(); }
+ int64_t MinBytes() const { return m_BytesHistogram.Min(); }
+ double MeanBytes() const { return m_BytesHistogram.Mean(); }
+ SampleSnapshot BytesSnapshot() const { return m_BytesHistogram.Snapshot(); }
+ double ByteRate1() { return m_BytesMeter.Rate1(); }
+ double ByteRate5() { return m_BytesMeter.Rate5(); }
+ double ByteRate15() { return m_BytesMeter.Rate15(); }
+ double ByteMeanRate() const { return m_BytesMeter.MeanRate(); }
+
+ struct Scope
+ {
+ Scope(OperationTiming& Outer);
+ ~Scope();
+
+ void Cancel();
+
+ private:
+ OperationTiming& m_Outer;
+ uint64_t m_StartTick;
+ };
+
+ void EmitSnapshot(std::string_view Tag, CbObjectWriter& Cbo);
+
+private:
+ Meter m_RequestMeter;
+ Meter m_BytesMeter;
+ Histogram m_RequestTimeHistogram;
+ Histogram m_BytesHistogram;
+};
+
+void EmitSnapshot(std::string_view Tag, OperationTiming& Stat, CbObjectWriter& Cbo);
+void EmitSnapshot(std::string_view Tag, const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor);
+void EmitSnapshot(std::string_view Tag, Meter& Stat, CbObjectWriter& Cbo);
+
+void EmitSnapshot(const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor);
+
+} // namespace zen::metrics
+
+namespace zen {
+
+extern void stats_forcelink();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/stream.h b/src/zencore/include/zencore/stream.h
new file mode 100644
index 000000000..9e4996249
--- /dev/null
+++ b/src/zencore/include/zencore/stream.h
@@ -0,0 +1,90 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/memory.h>
+#include <zencore/thread.h>
+
+#include <vector>
+
+namespace zen {
+
+/**
+ * Binary stream writer
+ */
+
+class BinaryWriter
+{
+public:
+ inline BinaryWriter() = default;
+ ~BinaryWriter() = default;
+
+ inline void Write(const void* DataPtr, size_t ByteCount)
+ {
+ Write(DataPtr, ByteCount, m_Offset);
+ m_Offset += ByteCount;
+ }
+
+ inline void Write(MemoryView Memory) { Write(Memory.GetData(), Memory.GetSize()); }
+ void Write(std::initializer_list<const MemoryView> Buffers);
+
+ inline uint64_t CurrentOffset() const { return m_Offset; }
+
+ inline const uint8_t* Data() const { return m_Buffer.data(); }
+ inline const uint8_t* GetData() const { return m_Buffer.data(); }
+ inline uint64_t Size() const { return m_Buffer.size(); }
+ inline uint64_t GetSize() const { return m_Buffer.size(); }
+ void Reset();
+
+ inline MemoryView GetView() const { return MemoryView(m_Buffer.data(), m_Offset); }
+ inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Buffer.data(), m_Offset); }
+
+private:
+ std::vector<uint8_t> m_Buffer;
+ uint64_t m_Offset = 0;
+
+ void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset);
+};
+
+inline MemoryView
+MakeMemoryView(const BinaryWriter& Stream)
+{
+ return MemoryView(Stream.Data(), Stream.Size());
+}
+
+/**
+ * Binary stream reader
+ */
+
+class BinaryReader
+{
+public:
+ inline BinaryReader(const void* Buffer, uint64_t Size) : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer)), m_BufferSize(Size) {}
+ inline BinaryReader(MemoryView Buffer)
+ : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer.GetData()))
+ , m_BufferSize(Buffer.GetSize())
+ {
+ }
+
+ inline void Read(void* DataPtr, size_t ByteCount)
+ {
+ memcpy(DataPtr, m_BufferBase + m_Offset, ByteCount);
+ m_Offset += ByteCount;
+ }
+
+ inline uint64_t Size() const { return m_BufferSize; }
+ inline uint64_t GetSize() const { return Size(); }
+ inline uint64_t CurrentOffset() const { return m_Offset; }
+ inline void Skip(size_t ByteCount) { m_Offset += ByteCount; };
+
+private:
+ const uint8_t* m_BufferBase;
+ uint64_t m_BufferSize;
+ uint64_t m_Offset = 0;
+};
+
+void stream_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h
new file mode 100644
index 000000000..ab111ff81
--- /dev/null
+++ b/src/zencore/include/zencore/string.h
@@ -0,0 +1,1115 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "intmath.h"
+#include "zencore.h"
+
+#include <stdint.h>
+#include <string.h>
+#include <charconv>
+#include <codecvt>
+#include <compare>
+#include <concepts>
+#include <optional>
+#include <span>
+#include <string_view>
+
+#include <type_traits>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+inline bool
+StringEquals(const char8_t* s1, const char* s2)
+{
+ return strcmp(reinterpret_cast<const char*>(s1), s2) == 0;
+}
+
+inline bool
+StringEquals(const char* s1, const char* s2)
+{
+ return strcmp(s1, s2) == 0;
+}
+
+inline size_t
+StringLength(const char* str)
+{
+ return strlen(str);
+}
+
+inline bool
+StringEquals(const wchar_t* s1, const wchar_t* s2)
+{
+ return wcscmp(s1, s2) == 0;
+}
+
+inline size_t
+StringLength(const wchar_t* str)
+{
+ return wcslen(str);
+}
+
+//////////////////////////////////////////////////////////////////////////
+// File name helpers
+//
+
+ZENCORE_API const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr);
+
+//////////////////////////////////////////////////////////////////////////
+// Text formatting of numbers
+//
+
+ZENCORE_API bool ToString(std::span<char> Buffer, uint64_t Num);
+ZENCORE_API bool ToString(std::span<char> Buffer, int64_t Num);
+
+struct TextNumBase
+{
+ inline const char* c_str() const { return m_Buffer; }
+ inline operator std::string_view() const { return std::string_view(m_Buffer); }
+
+protected:
+ char m_Buffer[24];
+};
+
+struct IntNum : public TextNumBase
+{
+ inline IntNum(UnsignedIntegral auto Number) { ToString(m_Buffer, uint64_t(Number)); }
+ inline IntNum(SignedIntegral auto Number) { ToString(m_Buffer, int64_t(Number)); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Quick-and-dirty string builder. Good enough for me, but contains traps
+// and not-quite-ideal behaviour especially when mixing character types etc
+//
+
+template<typename C>
+class StringBuilderImpl
+{
+public:
+ StringBuilderImpl() = default;
+ ZENCORE_API ~StringBuilderImpl();
+
+ StringBuilderImpl(const StringBuilderImpl&) = delete;
+ StringBuilderImpl(const StringBuilderImpl&&) = delete;
+ const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete;
+ const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete;
+
+ inline size_t AddUninitialized(size_t Count)
+ {
+ EnsureCapacity(Count);
+ const size_t OldCount = Size();
+ m_CurPos += Count;
+ return OldCount;
+ }
+
+ StringBuilderImpl& Append(C OneChar)
+ {
+ EnsureCapacity(1);
+
+ *m_CurPos++ = OneChar;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const std::string_view& String)
+ {
+ const size_t len = String.size();
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = String[i];
+
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const std::u8string_view& String)
+ {
+ const size_t len = String.size();
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = String[i];
+
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const char* NulTerminatedString)
+ {
+ size_t StringLen = StringLength(NulTerminatedString);
+
+ return AppendAscii({NulTerminatedString, StringLen});
+ }
+
+ inline StringBuilderImpl& Append(const char8_t* NulTerminatedString)
+ {
+ // This is super hacky and not fully functional - needs better
+ // solution
+ if constexpr (sizeof(C) == 1)
+ {
+ size_t len = StringLength((const char*)NulTerminatedString);
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = C(NulTerminatedString[i]);
+
+ m_CurPos += len;
+ }
+ else
+ {
+ ZEN_NOT_IMPLEMENTED();
+ }
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAsciiRange(const char* BeginString, const char* EndString)
+ {
+ EnsureCapacity(EndString - BeginString);
+
+ while (BeginString != EndString)
+ *m_CurPos++ = *BeginString++;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const C* NulTerminatedString)
+ {
+ size_t Len = StringLength(NulTerminatedString);
+
+ EnsureCapacity(Len);
+ memcpy(m_CurPos, NulTerminatedString, Len * sizeof(C));
+ m_CurPos += Len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const C* NulTerminatedString, size_t MaxChars)
+ {
+ size_t len = Min(MaxChars, StringLength(NulTerminatedString));
+
+ EnsureCapacity(len);
+ memcpy(m_CurPos, NulTerminatedString, len * sizeof(C));
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendRange(const C* BeginString, const C* EndString)
+ {
+ size_t Len = EndString - BeginString;
+
+ EnsureCapacity(Len);
+ memcpy(m_CurPos, BeginString, Len * sizeof(C));
+ m_CurPos += Len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const std::basic_string_view<C>& String)
+ {
+ return AppendRange(String.data(), String.data() + String.size());
+ }
+
+ inline StringBuilderImpl& AppendBool(bool v)
+ {
+ // This is a method instead of a << operator overload as the latter can
+ // easily get called with non-bool types like pointers. It is a very
+ // subtle behaviour that can cause bugs.
+ using namespace std::literals;
+ if (v)
+ {
+ return AppendAscii("true"sv);
+ }
+ return AppendAscii("false"sv);
+ }
+
+ inline void RemoveSuffix(uint32_t Count)
+ {
+ ZEN_ASSERT(Count <= Size());
+ m_CurPos -= Count;
+ }
+
+ inline const C* c_str() const
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline C* Data()
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline const C* Data() const
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline size_t Size() const { return m_CurPos - m_Base; }
+ inline bool IsDynamic() const { return m_IsDynamic; }
+ inline void Reset() { m_CurPos = m_Base; }
+
+ inline StringBuilderImpl& operator<<(uint64_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int64_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint32_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int32_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint16_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int16_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint8_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int8_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+
+ inline StringBuilderImpl& operator<<(const char* str) { return AppendAscii(str); }
+ inline StringBuilderImpl& operator<<(const std::string_view str) { return AppendAscii(str); }
+ inline StringBuilderImpl& operator<<(const std::u8string_view str) { return AppendAscii(str); }
+
+protected:
+ inline void Init(C* Base, size_t Capacity)
+ {
+ m_Base = m_CurPos = Base;
+ m_End = Base + Capacity;
+ }
+
+ inline void EnsureNulTerminated() const { *m_CurPos = '\0'; }
+
+ inline void EnsureCapacity(size_t ExtraRequired)
+ {
+ // precondition: we know the current buffer has enough capacity
+ // for the existing string including NUL terminator
+
+ if ((m_CurPos + ExtraRequired) < m_End)
+ return;
+
+ Extend(ExtraRequired);
+ }
+
+ ZENCORE_API void Extend(size_t ExtraCapacity);
+ ZENCORE_API void* AllocBuffer(size_t ByteCount);
+ ZENCORE_API void FreeBuffer(void* Buffer, size_t ByteCount);
+
+ ZENCORE_API [[noreturn]] void Fail(const char* FailReason); // note: throws exception
+
+ C* m_Base;
+ C* m_CurPos;
+ C* m_End;
+ bool m_IsDynamic = false;
+ bool m_IsExtendable = false;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+extern template class StringBuilderImpl<char>;
+
+inline StringBuilderImpl<char>&
+operator<<(StringBuilderImpl<char>& Builder, char Char)
+{
+ return Builder.Append(Char);
+}
+
+class StringBuilderBase : public StringBuilderImpl<char>
+{
+public:
+ inline StringBuilderBase(char* bufferPointer, size_t bufferCapacity) { Init(bufferPointer, bufferCapacity); }
+ inline ~StringBuilderBase() = default;
+
+ // Note that we don't need a terminator for the string_view so we avoid calling data() here
+ inline operator std::string_view() const { return std::string_view(m_Base, m_CurPos - m_Base); }
+ inline std::string_view ToView() const { return std::string_view(m_Base, m_CurPos - m_Base); }
+ inline std::string ToString() const { return std::string{Data(), Size()}; }
+
+ inline void AppendCodepoint(uint32_t cp)
+ {
+ if (cp < 0x80) // one octet
+ {
+ Append(static_cast<char8_t>(cp));
+ }
+ else if (cp < 0x800)
+ {
+ EnsureCapacity(2); // two octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 6) | 0xc0);
+ m_CurPos[1] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 2;
+ }
+ else if (cp < 0x10000)
+ {
+ EnsureCapacity(3); // three octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 12) | 0xe0);
+ m_CurPos[1] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80);
+ m_CurPos[2] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 3;
+ }
+ else
+ {
+ EnsureCapacity(4); // four octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 18) | 0xf0);
+ m_CurPos[1] = static_cast<char8_t>(((cp >> 12) & 0x3f) | 0x80);
+ m_CurPos[2] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80);
+ m_CurPos[3] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 4;
+ }
+ }
+};
+
+template<size_t N>
+class StringBuilder : public StringBuilderBase
+{
+public:
+ inline StringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) {}
+ inline ~StringBuilder() = default;
+
+private:
+ char m_StringBuffer[N];
+};
+
+template<size_t N>
+class ExtendableStringBuilder : public StringBuilderBase
+{
+public:
+ inline ExtendableStringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) { m_IsExtendable = true; }
+ inline ~ExtendableStringBuilder() = default;
+
+private:
+ char m_StringBuffer[N];
+};
+
+template<size_t N>
+class WriteToString : public ExtendableStringBuilder<N>
+{
+public:
+ template<typename... ArgTypes>
+ explicit WriteToString(ArgTypes&&... Args)
+ {
+ (*this << ... << std::forward<ArgTypes>(Args));
+ }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+extern template class StringBuilderImpl<wchar_t>;
+
+class WideStringBuilderBase : public StringBuilderImpl<wchar_t>
+{
+public:
+ inline WideStringBuilderBase(wchar_t* BufferPointer, size_t BufferCapacity) { Init(BufferPointer, BufferCapacity); }
+ inline ~WideStringBuilderBase() = default;
+
+ inline operator std::wstring_view() const { return std::wstring_view{Data(), Size()}; }
+ inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; }
+ inline std::wstring ToString() const { return std::wstring{Data(), Size()}; }
+
+ inline StringBuilderImpl& operator<<(const std::wstring_view str) { return Append((const wchar_t*)str.data(), str.size()); }
+ inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); }
+ using StringBuilderImpl:: operator<<;
+};
+
+template<size_t N>
+class WideStringBuilder : public WideStringBuilderBase
+{
+public:
+ inline WideStringBuilder() : WideStringBuilderBase(m_Buffer, N) {}
+ ~WideStringBuilder() = default;
+
+private:
+ wchar_t m_Buffer[N];
+};
+
+template<size_t N>
+class ExtendableWideStringBuilder : public WideStringBuilderBase
+{
+public:
+ inline ExtendableWideStringBuilder() : WideStringBuilderBase(m_Buffer, N) { m_IsExtendable = true; }
+ ~ExtendableWideStringBuilder() = default;
+
+private:
+ wchar_t m_Buffer[N];
+};
+
+template<size_t N>
+class WriteToWideString : public ExtendableWideStringBuilder<N>
+{
+public:
+ template<typename... ArgTypes>
+ explicit WriteToWideString(ArgTypes&&... Args)
+ {
+ (*this << ... << Forward<ArgTypes>(Args));
+ }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out);
+void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out);
+void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out);
+std::wstring Utf8ToWide(const std::string_view& wstr);
+
+void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out);
+std::string WideToUtf8(const wchar_t* wstr);
+void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out);
+std::string WideToUtf8(const std::wstring_view Wstr);
+
+inline uint8_t
+Char2Nibble(char c)
+{
+ if (c >= '0' && c <= '9')
+ {
+ return uint8_t(c - '0');
+ }
+ if (c >= 'a' && c <= 'f')
+ {
+ return uint8_t(c - 'a' + 10);
+ }
+ if (c >= 'A' && c <= 'F')
+ {
+ return uint8_t(c - 'A' + 10);
+ }
+ return uint8_t(0xff);
+};
+
+static constexpr const char HexChars[] = "0123456789abcdef";
+
+/// <summary>
+/// Parse hex string into a byte buffer
+/// </summary>
+/// <param name="string">Input string</param>
+/// <param name="characterCount">Number of characters in string</param>
+/// <param name="outPtr">Pointer to output buffer</param>
+/// <returns>true if the input consisted of all valid hexadecimal characters</returns>
+
+inline bool
+ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr)
+{
+ ZEN_ASSERT((CharacterCount & 1) == 0);
+
+ uint8_t allBits = 0;
+
+ while (CharacterCount)
+ {
+ uint8_t n0 = Char2Nibble(InputString[0]);
+ uint8_t n1 = Char2Nibble(InputString[1]);
+
+ allBits |= n0 | n1;
+
+ *OutPtr = (n0 << 4) | n1;
+
+ OutPtr += 1;
+ InputString += 2;
+ CharacterCount -= 2;
+ }
+
+ return (allBits & 0x80) == 0;
+}
+
+inline void
+ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString)
+{
+ while (ByteCount--)
+ {
+ uint8_t byte = *InputData++;
+
+ *OutString++ = HexChars[byte >> 4];
+ *OutString++ = HexChars[byte & 15];
+ }
+}
+
+inline bool
+ParseHexNumber(const char* InputString, size_t CharacterCount, uint8_t* OutPtr)
+{
+ ZEN_ASSERT((CharacterCount & 1) == 0);
+
+ uint8_t allBits = 0;
+
+ InputString += CharacterCount;
+ while (CharacterCount)
+ {
+ InputString -= 2;
+ uint8_t n0 = Char2Nibble(InputString[0]);
+ uint8_t n1 = Char2Nibble(InputString[1]);
+
+ allBits |= n0 | n1;
+
+ *OutPtr = (n0 << 4) | n1;
+
+ OutPtr += 1;
+ CharacterCount -= 2;
+ }
+
+ return (allBits & 0x80) == 0;
+}
+
+inline void
+ToHexNumber(const uint8_t* InputData, size_t ByteCount, char* OutString)
+{
+ InputData += ByteCount;
+ while (ByteCount--)
+ {
+ uint8_t byte = *(--InputData);
+
+ *OutString++ = HexChars[byte >> 4];
+ *OutString++ = HexChars[byte & 15];
+ }
+}
+
+/// <summary>
+/// Generates a hex number from a pointer to an integer type, this formats the number in the correct order for a hexadecimal number
+/// </summary>
+/// <param name="Value">Integer value type</param>
+/// <param name="outString">Output buffer where resulting string is written</param>
+void
+ToHexNumber(UnsignedIntegral auto Value, char* OutString)
+{
+ ToHexNumber((const uint8_t*)&Value, sizeof(Value), OutString);
+ OutString[sizeof(Value) * 2] = 0;
+}
+
+/// <summary>
+/// Parse hex number string into a value, this formats the number in the correct order for a hexadecimal number
+/// </summary>
+/// <param name="string">Input string</param>
+/// <param name="characterCount">Number of characters in string</param>
+/// <param name="OutValue">Pointer to output value</param>
+/// <returns>true if the input consisted of all valid hexadecimal characters</returns>
+bool
+ParseHexNumber(const std::string HexString, UnsignedIntegral auto& OutValue)
+{
+ return ParseHexNumber(HexString.c_str(), sizeof(OutValue) * 2, (uint8_t*)&OutValue);
+}
+
+//////////////////////////////////////////////////////////////////////////
+// Format numbers for humans
+//
+
+ZENCORE_API size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer);
+ZENCORE_API size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer);
+ZENCORE_API size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer);
+ZENCORE_API size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer);
+ZENCORE_API size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer);
+
+struct NiceBase
+{
+ inline const char* c_str() const { return m_Buffer; }
+ inline operator std::string_view() const { return std::string_view(m_Buffer); }
+
+protected:
+ char m_Buffer[16];
+};
+
+struct NiceNum : public NiceBase
+{
+ inline NiceNum(uint64_t Num) { NiceNumToBuffer(Num, m_Buffer); }
+};
+
+struct NiceBytes : public NiceBase
+{
+ inline NiceBytes(uint64_t Num) { NiceBytesToBuffer(Num, m_Buffer); }
+};
+
+struct NiceByteRate : public NiceBase
+{
+ inline NiceByteRate(uint64_t Bytes, uint64_t TimeMilliseconds) { NiceByteRateToBuffer(Bytes, TimeMilliseconds, m_Buffer); }
+};
+
+struct NiceLatencyNs : public NiceBase
+{
+ inline NiceLatencyNs(uint64_t Milliseconds) { NiceLatencyNsToBuffer(Milliseconds, m_Buffer); }
+};
+
+struct NiceTimeSpanMs : public NiceBase
+{
+ inline NiceTimeSpanMs(uint64_t Milliseconds) { NiceTimeSpanMsToBuffer(Milliseconds, m_Buffer); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+inline std::string
+NiceRate(uint64_t Num, uint32_t DurationMilliseconds, const char* Unit = "B")
+{
+ char Buffer[32];
+
+ if (DurationMilliseconds)
+ {
+ // Leave a little of 'Buffer' for the "Unit/s" suffix
+ std::span<char> BufferSpan(Buffer, sizeof(Buffer) - 8);
+ NiceNumToBuffer(Num * 1000 / DurationMilliseconds, BufferSpan);
+ }
+ else
+ {
+ strcpy(Buffer, "0");
+ }
+
+ strncat(Buffer, Unit, 4);
+ strcat(Buffer, "/s");
+
+ return Buffer;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template<Integral T>
+std::optional<T>
+ParseInt(const std::string_view& Input)
+{
+ T Out = 0;
+ const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out);
+ if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range)
+ {
+ return std::nullopt;
+ }
+ return Out;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+constexpr uint32_t
+HashStringDjb2(const std::string_view& InString)
+{
+ uint32_t HashValue = 5381;
+
+ for (int CurChar : InString)
+ {
+ HashValue = HashValue * 33 + CurChar;
+ }
+
+ return HashValue;
+}
+
+constexpr uint32_t
+HashStringAsLowerDjb2(const std::string_view& InString)
+{
+ uint32_t HashValue = 5381;
+
+ for (uint8_t CurChar : InString)
+ {
+ CurChar -= ((CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic
+ HashValue = HashValue * 33 + CurChar;
+ }
+
+ return HashValue;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+inline std::string
+ToLower(const std::string_view& InString)
+{
+ std::string Out(InString);
+
+ for (char& CurChar : Out)
+ {
+ CurChar -= (uint8_t(CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic
+ }
+
+ return Out;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template<typename Fn>
+uint32_t
+ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
+{
+ const char* It = Str.data();
+ const char* End = It + Str.length();
+ uint32_t Count = 0;
+
+ while (It != End)
+ {
+ if (*It == Delim)
+ {
+ It++;
+ continue;
+ }
+
+ std::string_view Remaining{It, size_t(ptrdiff_t(End - It))};
+ size_t Idx = Remaining.find(Delim, 0);
+
+ if (Idx == std::string_view::npos)
+ {
+ Idx = Remaining.size();
+ }
+
+ Count++;
+ std::string_view Token{It, Idx};
+ if (!Func(Token))
+ {
+ break;
+ }
+
+ It = It + Idx;
+ }
+
+ return Count;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+inline int32_t
+StrCaseCompare(const char* Lhs, const char* Rhs, int64_t Length = -1)
+{
+ // A helper for cross-platform case-insensitive string comparison.
+#if ZEN_PLATFORM_WINDOWS
+ return (Length < 0) ? _stricmp(Lhs, Rhs) : _strnicmp(Lhs, Rhs, size_t(Length));
+#else
+ return (Length < 0) ? strcasecmp(Lhs, Rhs) : strncasecmp(Lhs, Rhs, size_t(Length));
+#endif
+}
+
+/**
+ * @brief
+ * Helper function to implement case sensitive spaceship operator for strings.
+ * MacOS clang version we use does not implement <=> for std::string
+ * @param Lhs string
+ * @param Rhs string
+ * @return std::strong_ordering indicating relationship between Lhs and Rhs
+ */
+inline auto
+caseSensitiveCompareStrings(const std::string& Lhs, const std::string& Rhs)
+{
+ int r = Lhs.compare(Rhs);
+ return r == 0 ? std::strong_ordering::equal : r < 0 ? std::strong_ordering::less : std::strong_ordering::greater;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+/**
+ * ASCII character bitset useful for fast and readable parsing
+ *
+ * Entirely constexpr. Works with both wide and narrow strings.
+ *
+ * Example use cases:
+ *
+ * constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n");
+ * bool bIsWhitespace = WhitespaceCharacters.Test(MyChar);
+ * const char* HelloWorld = AsciiSet::Skip(" \t\tHello world!", WhitespaceCharacters);
+ *
+ * constexpr AsciiSet XmlEscapeChars("&<>\"'");
+ * check(AsciiSet::HasNone(EscapedXmlString, XmlEscapeChars));
+ *
+ * constexpr AsciiSet Delimiters(".:;");
+ * const TCHAR* DelimiterOrEnd = AsciiSet::FindFirstOrEnd(PrefixedName, Delimiters);
+ * FString Prefix(PrefixedName, DelimiterOrEnd - PrefixedName);
+ *
+ * constexpr AsciiSet Slashes("/\\");
+ * const TCHAR* SlashOrEnd = AsciiSet::FindLastOrEnd(PathName, Slashes);
+ * const TCHAR* FileName = *SlashOrEnd ? SlashOrEnd + 1 : PathName;
+ */
+class AsciiSet
+{
+public:
+ template<typename CharType, int N>
+ constexpr AsciiSet(const CharType (&Chars)[N]) : AsciiSet(StringToBitset(Chars))
+ {
+ }
+
+ /** Returns true if a character is part of the set */
+ template<typename CharType>
+ constexpr inline bool Contains(CharType Char) const
+ {
+ using UnsignedCharType = typename std::make_unsigned<CharType>::type;
+
+ return !!TestImpl((UnsignedCharType)Char);
+ }
+
+ /** Returns non-zero if a character is part of the set. Prefer Contains() to avoid VS2019 conversion warnings. */
+ template<typename CharType>
+ constexpr inline uint64_t Test(CharType Char) const
+ {
+ using UnsignedCharType = typename std::make_unsigned<CharType>::type;
+
+ return TestImpl((UnsignedCharType)Char);
+ }
+
+ /** Create new set with specified character in it */
+ constexpr inline AsciiSet operator+(char Char) const
+ {
+ using UnsignedCharType = typename std::make_unsigned<char>::type;
+
+ InitData Bitset = {LoMask, HiMask};
+ SetImpl(Bitset, (UnsignedCharType)Char);
+ return AsciiSet(Bitset);
+ }
+
+ /** Create new set containing inverse set of characters - likely including null-terminator */
+ constexpr inline AsciiSet operator~() const { return AsciiSet(~LoMask, ~HiMask); }
+
+ ////////// Algorithms for C strings //////////
+
+ /** Find first character of string inside set or end pointer. Never returns null. */
+ template<class CharType>
+ static constexpr const CharType* FindFirstOrEnd(const CharType* Str, AsciiSet Set)
+ {
+ for (AsciiSet SetOrNil(Set.LoMask | NilMask, Set.HiMask); !SetOrNil.Test(*Str); ++Str)
+ ;
+
+ return Str;
+ }
+
+ /** Find last character of string inside set or end pointer. Never returns null. */
+ template<class CharType>
+ static constexpr const CharType* FindLastOrEnd(const CharType* Str, AsciiSet Set)
+ {
+ const CharType* Last = FindFirstOrEnd(Str, Set);
+
+ for (const CharType* It = Last; *It; It = FindFirstOrEnd(It + 1, Set))
+ {
+ Last = It;
+ }
+
+ return Last;
+ }
+
+ /** Find first character of string outside of set. Never returns null. */
+ template<typename CharType>
+ static constexpr const CharType* Skip(const CharType* Str, AsciiSet Set)
+ {
+ while (Set.Contains(*Str))
+ {
+ ++Str;
+ }
+
+ return Str;
+ }
+
+ /** Test if string contains any character in set */
+ template<typename CharType>
+ static constexpr bool HasAny(const CharType* Str, AsciiSet Set)
+ {
+ return *FindFirstOrEnd(Str, Set) != '\0';
+ }
+
+ /** Test if string contains no character in set */
+ template<typename CharType>
+ static constexpr bool HasNone(const CharType* Str, AsciiSet Set)
+ {
+ return *FindFirstOrEnd(Str, Set) == '\0';
+ }
+
+ /** Test if string contains any character outside of set */
+ template<typename CharType>
+ static constexpr bool HasOnly(const CharType* Str, AsciiSet Set)
+ {
+ return *Skip(Str, Set) == '\0';
+ }
+
+ ////////// Algorithms for string types like std::string_view and std::string //////////
+
+ /** Get initial substring with all characters in set */
+ template<class StringType>
+ static constexpr StringType FindPrefixWith(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Forward, EInclude::Members, EKeep::Head>(Str, Set);
+ }
+
+ /** Get initial substring with no characters in set */
+ template<class StringType>
+ static constexpr StringType FindPrefixWithout(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Head>(Str, Set);
+ }
+
+ /** Trim initial characters in set */
+ template<class StringType>
+ static constexpr StringType TrimPrefixWith(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Forward, EInclude::Members, EKeep::Tail>(Str, Set);
+ }
+
+ /** Trim initial characters not in set */
+ template<class StringType>
+ static constexpr StringType TrimPrefixWithout(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Tail>(Str, Set);
+ }
+
+ /** Get trailing substring with all characters in set */
+ template<class StringType>
+ static constexpr StringType FindSuffixWith(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Reverse, EInclude::Members, EKeep::Tail>(Str, Set);
+ }
+
+ /** Get trailing substring with no characters in set */
+ template<class StringType>
+ static constexpr StringType FindSuffixWithout(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Tail>(Str, Set);
+ }
+
+ /** Trim trailing characters in set */
+ template<class StringType>
+ static constexpr StringType TrimSuffixWith(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Reverse, EInclude::Members, EKeep::Head>(Str, Set);
+ }
+
+ /** Trim trailing characters not in set */
+ template<class StringType>
+ static constexpr StringType TrimSuffixWithout(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Head>(Str, Set);
+ }
+
+ /** Test if string contains any character in set */
+ template<class StringType>
+ static constexpr bool HasAny(const StringType& Str, AsciiSet Set)
+ {
+ return !HasNone(Str, Set);
+ }
+
+ /** Test if string contains no character in set */
+ template<class StringType>
+ static constexpr bool HasNone(const StringType& Str, AsciiSet Set)
+ {
+ uint64_t Match = 0;
+ for (auto Char : Str)
+ {
+ Match |= Set.Test(Char);
+ }
+ return Match == 0;
+ }
+
+ /** Test if string contains any character outside of set */
+ template<class StringType>
+ static constexpr bool HasOnly(const StringType& Str, AsciiSet Set)
+ {
+ auto End = Str.data() + Str.size();
+ return FindFirst<EInclude::Members>(Set, Str.data(), End) == End;
+ }
+
+private:
+ enum class EDir
+ {
+ Forward,
+ Reverse
+ };
+ enum class EInclude
+ {
+ Members,
+ NonMembers
+ };
+ enum class EKeep
+ {
+ Head,
+ Tail
+ };
+
+ template<EInclude Include, typename CharType>
+ static constexpr const CharType* FindFirst(AsciiSet Set, const CharType* It, const CharType* End)
+ {
+ for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); ++It)
+ ;
+ return It;
+ }
+
+ template<EInclude Include, typename CharType>
+ static constexpr const CharType* FindLast(AsciiSet Set, const CharType* It, const CharType* End)
+ {
+ for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); --It)
+ ;
+ return It;
+ }
+
+ template<EDir Dir, EInclude Include, EKeep Keep, class StringType>
+ static constexpr StringType Scan(const StringType& Str, AsciiSet Set)
+ {
+ auto Begin = Str.data();
+ auto End = Begin + Str.size();
+ auto It = Dir == EDir::Forward ? FindFirst<Include>(Set, Begin, End) : FindLast<Include>(Set, End - 1, Begin - 1) + 1;
+
+ return Keep == EKeep::Head ? StringType(Begin, static_cast<int32_t>(It - Begin)) : StringType(It, static_cast<int32_t>(End - It));
+ }
+
+ // Work-around for constexpr limitations
+ struct InitData
+ {
+ uint64_t Lo, Hi;
+ };
+ static constexpr uint64_t NilMask = uint64_t(1) << '\0';
+
+ static constexpr inline void SetImpl(InitData& Bitset, uint32_t Char)
+ {
+ uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0);
+ uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1);
+ uint64_t Bit = uint64_t(1) << uint8_t(Char & 0x3f);
+
+ Bitset.Lo |= Bit & IsLo;
+ Bitset.Hi |= Bit & IsHi;
+ }
+
+ constexpr inline uint64_t TestImpl(uint32_t Char) const
+ {
+ uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0);
+ uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1);
+ uint64_t Bit = uint64_t(1) << (Char & 0x3f);
+
+ return (Bit & IsLo & LoMask) | (Bit & IsHi & HiMask);
+ }
+
+ template<typename CharType, int N>
+ static constexpr InitData StringToBitset(const CharType (&Chars)[N])
+ {
+ using UnsignedCharType = typename std::make_unsigned<CharType>::type;
+
+ InitData Bitset = {0, 0};
+ for (int I = 0; I < N - 1; ++I)
+ {
+ SetImpl(Bitset, UnsignedCharType(Chars[I]));
+ }
+
+ return Bitset;
+ }
+
+ constexpr AsciiSet(InitData Bitset) : LoMask(Bitset.Lo), HiMask(Bitset.Hi) {}
+
+ constexpr AsciiSet(uint64_t Lo, uint64_t Hi) : LoMask(Lo), HiMask(Hi) {}
+
+ uint64_t LoMask, HiMask;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+void string_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/testing.h b/src/zencore/include/zencore/testing.h
new file mode 100644
index 000000000..a00ee3166
--- /dev/null
+++ b/src/zencore/include/zencore/testing.h
@@ -0,0 +1,67 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <memory>
+
+#ifndef ZEN_TEST_WITH_RUNNER
+# define ZEN_TEST_WITH_RUNNER 0
+#endif
+
+#if ZEN_TEST_WITH_RUNNER
+# define DOCTEST_CONFIG_IMPLEMENT
+#endif
+
+#if ZEN_WITH_TESTS
+# include <doctest/doctest.h>
+inline auto
+Approx(auto Value)
+{
+ return doctest::Approx(Value);
+}
+#endif
+
+/**
+ * Test runner helper
+ *
+ * This acts as a thin layer between the test app and the test
+ * framework, which is used to customize configuration logic
+ * and to set up logging.
+ *
+ * If you don't want to implement custom setup then the
+ * ZEN_RUN_TESTS macro can be used instead.
+ */
+
+#if ZEN_WITH_TESTS
+namespace zen::testing {
+
+class TestRunner
+{
+public:
+ TestRunner();
+ ~TestRunner();
+
+ int ApplyCommandLine(int argc, char const* const* argv);
+ int Run();
+
+private:
+ struct Impl;
+
+ std::unique_ptr<Impl> m_Impl;
+};
+
+# define ZEN_RUN_TESTS(argC, argV) \
+ [&] { \
+ zen::testing::TestRunner Runner; \
+ Runner.ApplyCommandLine(argC, argV); \
+ return Runner.Run(); \
+ }()
+
+} // namespace zen::testing
+#endif
+
+#if ZEN_TEST_WITH_RUNNER
+# undef DOCTEST_CONFIG_IMPLEMENT
+#endif
diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h
new file mode 100644
index 000000000..04648c6de
--- /dev/null
+++ b/src/zencore/include/zencore/testutils.h
@@ -0,0 +1,32 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <filesystem>
+
+namespace zen {
+
+std::filesystem::path CreateTemporaryDirectory();
+
+class ScopedTemporaryDirectory
+{
+public:
+ explicit ScopedTemporaryDirectory(std::filesystem::path Directory);
+ ScopedTemporaryDirectory();
+ ~ScopedTemporaryDirectory();
+
+ std::filesystem::path& Path() { return m_RootPath; }
+
+private:
+ std::filesystem::path m_RootPath;
+};
+
+struct ScopedCurrentDirectoryChange
+{
+ std::filesystem::path OldPath{std::filesystem::current_path()};
+
+ ScopedCurrentDirectoryChange() { std::filesystem::current_path(CreateTemporaryDirectory()); }
+ ~ScopedCurrentDirectoryChange() { std::filesystem::current_path(OldPath); }
+};
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h
new file mode 100644
index 000000000..a9c96d422
--- /dev/null
+++ b/src/zencore/include/zencore/thread.h
@@ -0,0 +1,273 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <atomic>
+#include <filesystem>
+#include <shared_mutex>
+#include <string_view>
+#include <vector>
+
+namespace zen {
+
+void SetCurrentThreadName(std::string_view ThreadName);
+
+/**
+ * Reader-writer lock
+ *
+ * - A single thread may hold an exclusive lock at any given moment
+ *
+ * - Multiple threads may hold shared locks, but only if no thread has
+ * acquired an exclusive lock
+ */
+class RwLock
+{
+public:
+ ZENCORE_API void AcquireShared();
+ ZENCORE_API void ReleaseShared();
+
+ ZENCORE_API void AcquireExclusive();
+ ZENCORE_API void ReleaseExclusive();
+
+ struct SharedLockScope
+ {
+ SharedLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireShared(); }
+ ~SharedLockScope() { ReleaseNow(); }
+
+ void ReleaseNow()
+ {
+ if (m_Lock)
+ {
+ m_Lock->ReleaseShared();
+ m_Lock = nullptr;
+ }
+ }
+
+ private:
+ RwLock* m_Lock;
+ };
+
+ struct ExclusiveLockScope
+ {
+ ExclusiveLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireExclusive(); }
+ ~ExclusiveLockScope() { ReleaseNow(); }
+
+ void ReleaseNow()
+ {
+ if (m_Lock)
+ {
+ m_Lock->ReleaseExclusive();
+ m_Lock = nullptr;
+ }
+ }
+
+ private:
+ RwLock* m_Lock;
+ };
+
+private:
+ std::shared_mutex m_Mutex;
+};
+
+/** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore')
+ */
+class Event
+{
+public:
+ ZENCORE_API Event();
+ ZENCORE_API ~Event();
+
+ Event(Event&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; }
+
+ Event(const Event& Rhs) = delete;
+ Event& operator=(const Event& Rhs) = delete;
+
+ inline Event& operator=(Event&& Rhs) noexcept
+ {
+ std::swap(m_EventHandle, Rhs.m_EventHandle);
+ return *this;
+ }
+
+ ZENCORE_API void Set();
+ ZENCORE_API void Reset();
+ ZENCORE_API bool Wait(int TimeoutMs = -1);
+ ZENCORE_API void Close();
+
+protected:
+ explicit Event(void* EventHandle) : m_EventHandle(EventHandle) {}
+
+ void* m_EventHandle = nullptr;
+};
+
+/** Basic abstraction of an IPC mechanism (aka 'binary semaphore')
+ */
+class NamedEvent
+{
+public:
+ NamedEvent() = default;
+ ZENCORE_API explicit NamedEvent(std::string_view EventName);
+ ZENCORE_API ~NamedEvent();
+ ZENCORE_API void Close();
+ ZENCORE_API void Set();
+ ZENCORE_API bool Wait(int TimeoutMs = -1);
+
+ NamedEvent(NamedEvent&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; }
+
+ inline NamedEvent& operator=(NamedEvent&& Rhs) noexcept
+ {
+ std::swap(m_EventHandle, Rhs.m_EventHandle);
+ return *this;
+ }
+
+protected:
+ void* m_EventHandle = nullptr;
+
+private:
+ NamedEvent(const NamedEvent& Rhs) = delete;
+ NamedEvent& operator=(const NamedEvent& Rhs) = delete;
+};
+
+/** Basic abstraction of a named (system wide) mutex primitive
+ */
+class NamedMutex
+{
+public:
+ ~NamedMutex();
+
+ ZENCORE_API [[nodiscard]] bool Create(std::string_view MutexName);
+
+ ZENCORE_API static bool Exists(std::string_view MutexName);
+
+private:
+ void* m_MutexHandle = nullptr;
+};
+
+/**
+ * Downward counter of type std::ptrdiff_t which can be used to synchronize threads
+ */
+class Latch
+{
+public:
+ Latch(std::ptrdiff_t Count) : Counter(Count) {}
+
+ void CountDown()
+ {
+ std::ptrdiff_t Old = Counter.fetch_sub(1);
+ if (Old == 1)
+ {
+ Complete.Set();
+ }
+ }
+
+ std::ptrdiff_t Remaining() const { return Counter.load(); }
+
+ // If you want to add dynamic count, make sure to set the initial counter to 1
+ // and then do a CountDown() just before wait to not trigger the event causing
+ // false positive completion results.
+ void AddCount(std::ptrdiff_t Count)
+ {
+ std::atomic_ptrdiff_t Old = Counter.fetch_add(Count);
+ ZEN_ASSERT_SLOW(Old > 0);
+ }
+
+ bool Wait(int TimeoutMs = -1)
+ {
+ std::ptrdiff_t Old = Counter.load();
+ if (Old == 0)
+ {
+ return true;
+ }
+ return Complete.Wait(TimeoutMs);
+ }
+
+private:
+ std::atomic_ptrdiff_t Counter;
+ Event Complete;
+};
+
+/** Basic process abstraction
+ */
+class ProcessHandle
+{
+public:
+ ZENCORE_API ProcessHandle();
+
+ ProcessHandle(const ProcessHandle&) = delete;
+ ProcessHandle& operator=(const ProcessHandle&) = delete;
+
+ ZENCORE_API ~ProcessHandle();
+
+ ZENCORE_API void Initialize(int Pid);
+ ZENCORE_API void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle
+ ZENCORE_API [[nodiscard]] bool IsRunning() const;
+ ZENCORE_API [[nodiscard]] bool IsValid() const;
+ ZENCORE_API bool Wait(int TimeoutMs = -1);
+ ZENCORE_API void Terminate(int ExitCode);
+ ZENCORE_API void Reset();
+ [[nodiscard]] inline int Pid() const { return m_Pid; }
+
+private:
+ void* m_ProcessHandle = nullptr;
+ int m_Pid = 0;
+};
+
+/** Basic process creation
+ */
+struct CreateProcOptions
+{
+ enum
+ {
+ Flag_NewConsole = 1 << 0,
+ Flag_Elevated = 1 << 1,
+ Flag_Unelevated = 1 << 2,
+ };
+
+ const std::filesystem::path* WorkingDirectory = nullptr;
+ uint32_t Flags = 0;
+};
+
+#if ZEN_PLATFORM_WINDOWS
+using CreateProcResult = void*; // handle to the process
+#else
+using CreateProcResult = int32_t; // pid
+#endif
+
+ZENCORE_API CreateProcResult CreateProc(const std::filesystem::path& Executable,
+ std::string_view CommandLine, // should also include arg[0] (executable name)
+ const CreateProcOptions& Options = {});
+
+/** Process monitor - monitors a list of running processes via polling
+
+ Intended to be used to monitor a set of "sponsor" processes, where
+ we need to determine when none of them remain alive
+
+ */
+
+class ProcessMonitor
+{
+public:
+ ProcessMonitor();
+ ~ProcessMonitor();
+
+ ZENCORE_API bool IsRunning();
+ ZENCORE_API void AddPid(int Pid);
+ ZENCORE_API bool IsActive() const;
+
+private:
+ using HandleType = void*;
+
+ mutable RwLock m_Lock;
+ std::vector<HandleType> m_ProcessHandles;
+};
+
+ZENCORE_API bool IsProcessRunning(int pid);
+ZENCORE_API int GetCurrentProcessId();
+ZENCORE_API int GetCurrentThreadId();
+
+ZENCORE_API void Sleep(int ms);
+
+void thread_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/timer.h b/src/zencore/include/zencore/timer.h
new file mode 100644
index 000000000..e4ddc3505
--- /dev/null
+++ b/src/zencore/include/zencore/timer.h
@@ -0,0 +1,58 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#if ZEN_COMPILER_MSC
+# include <intrin.h>
+#elif ZEN_ARCH_X64
+# include <x86intrin.h>
+#endif
+
+#include <stdint.h>
+
+namespace zen {
+
+// High frequency timers
+
+ZENCORE_API uint64_t GetHifreqTimerValue();
+ZENCORE_API uint64_t GetHifreqTimerFrequency();
+ZENCORE_API double GetHifreqTimerToSeconds();
+ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init
+
+class Stopwatch
+{
+public:
+ inline Stopwatch() : m_StartValue(GetHifreqTimerValue()) {}
+
+ inline uint64_t GetElapsedTimeMs() const { return (GetHifreqTimerValue() - m_StartValue) * 1'000 / GetHifreqTimerFrequency(); }
+ inline uint64_t GetElapsedTimeUs() const { return (GetHifreqTimerValue() - m_StartValue) * 1'000'000 / GetHifreqTimerFrequency(); }
+ inline uint64_t GetElapsedTicks() const { return GetHifreqTimerValue() - m_StartValue; }
+ inline void Reset() { m_StartValue = GetHifreqTimerValue(); }
+
+ static inline uint64_t GetElapsedTimeMs(uint64_t Ticks) { return Ticks * 1'000 / GetHifreqTimerFrequency(); }
+ static inline uint64_t GetElapsedTimeUs(uint64_t Ticks) { return Ticks * 1'000'000 / GetHifreqTimerFrequency(); }
+
+private:
+ uint64_t m_StartValue;
+};
+
+// Low frequency timers
+
+namespace detail {
+ extern ZENCORE_API uint64_t g_LofreqTimerValue;
+} // namespace detail
+
+inline uint64_t
+GetLofreqTimerValue()
+{
+ return detail::g_LofreqTimerValue;
+}
+
+ZENCORE_API void UpdateLofreqTimerValue();
+ZENCORE_API uint64_t GetLofreqTimerFrequency();
+
+void timer_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h
new file mode 100644
index 000000000..0af490f23
--- /dev/null
+++ b/src/zencore/include/zencore/trace.h
@@ -0,0 +1,36 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+/* clang-format off */
+
+#include <zencore/zencore.h>
+
+#if ZEN_WITH_TRACE
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#if !defined(TRACE_IMPLEMENT)
+# define TRACE_IMPLEMENT 0
+#endif
+#include <trace.h>
+#undef TRACE_IMPLEMENT
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#define ZEN_TRACE_CPU(x) TRACE_CPU_SCOPE(x)
+
+enum class TraceType
+{
+ File,
+ Network,
+ None
+};
+
+void TraceInit(const char* HostOrPath, TraceType Type);
+
+#else
+
+#define ZEN_TRACE_CPU(x)
+
+#endif // ZEN_WITH_TRACE
+
+/* clang-format on */
diff --git a/src/zencore/include/zencore/uid.h b/src/zencore/include/zencore/uid.h
new file mode 100644
index 000000000..9659f5893
--- /dev/null
+++ b/src/zencore/include/zencore/uid.h
@@ -0,0 +1,87 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+#include <compare>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/** Object identifier
+
+ Can be used as a GUID essentially, but is more compact (12 bytes) and as such
+ is more susceptible to collisions than a 16-byte GUID but also I don't expect
+ the population to be large so in practice the risk should be minimal due to
+ how the identifiers work.
+
+ Similar in spirit to MongoDB ObjectId
+
+ When serialized, object identifiers generated in a given session in sequence
+ will sort in chronological order since the timestamp is in the MSB in big
+ endian format. This makes it suitable as a database key since most indexing
+ structures work better when keys are inserted in lexicographically
+ increasing order.
+
+ The current layout is basically:
+
+ |----------------|----------------|----------------|
+ | timestamp | serial # | run id |
+ |----------------|----------------|----------------|
+ MSB LSB
+
+ - Timestamp is a unsigned 32-bit value (seconds since 00:00:00 Jan 1 2021)
+ - Serial # is another unsigned 32-bit value which is assigned a (strong)
+ random number at initialization time which is incremented when a new Oid
+ is generated
+ - The run id is generated from a strong random number generator
+ at initialization time and stays fixed for the duration of the program
+
+ Timestamp and serial are stored in memory in such a way that they can be
+ ordered lexicographically. I.e they are in big-endian byte order.
+
+ NOTE: The information above is only meant to explain the properties of
+ the identifiers. Client code should simply treat the identifier as an
+ opaque value and may not make any assumptions on the structure, as there
+ may be other ways of generating the identifiers in the future if an
+ application benefits.
+
+ */
+
+struct Oid
+{
+ static const int StringLength = 24;
+ typedef char String_t[StringLength + 1];
+
+ static void Initialize();
+ [[nodiscard]] static Oid NewOid();
+
+ const Oid& Generate();
+ [[nodiscard]] static Oid FromHexString(const std::string_view String);
+ StringBuilderBase& ToString(StringBuilderBase& OutString) const;
+ void ToString(char OutString[StringLength]);
+ [[nodiscard]] static Oid FromMemory(const void* Ptr);
+
+ auto operator<=>(const Oid& rhs) const = default;
+ [[nodiscard]] inline operator bool() const { return *this != Zero; }
+
+ static const Oid Zero; // Min (can be used to signify a "null" value, or for open range queries)
+ static const Oid Max; // Max (can be used for open range queries)
+
+ struct Hasher
+ {
+ size_t operator()(const Oid& id) const
+ {
+ const size_t seed = id.OidBits[0];
+ return ((seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1])) | (uint64_t(id.OidBits[2]) << 32);
+ }
+ };
+
+ // You should not assume anything about these words
+ uint32_t OidBits[3];
+};
+
+extern void uid_forcelink();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/varint.h b/src/zencore/include/zencore/varint.h
new file mode 100644
index 000000000..e57e1d497
--- /dev/null
+++ b/src/zencore/include/zencore/varint.h
@@ -0,0 +1,277 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "intmath.h"
+
+#include <algorithm>
+
+namespace zen {
+
+// Variable-Length Integer Encoding
+//
+// ZigZag encoding is used to convert signed integers into unsigned integers in a way that allows
+// integers with a small magnitude to have a smaller encoded representation.
+//
+// An unsigned integer is encoded into 1-9 bytes based on its magnitude. The first byte indicates
+// how many additional bytes are used by the number of leading 1-bits that it has. The additional
+// bytes are stored in big endian order, and the most significant bits of the value are stored in
+// the remaining bits in the first byte. The encoding of the first byte allows the reader to skip
+// over the encoded integer without consuming its bytes individually.
+//
+// Encoded unsigned integers sort the same in a byte-wise comparison as when their decoded values
+// are compared. The same property does not hold for signed integers due to ZigZag encoding.
+//
+// 32-bit inputs encode to 1-5 bytes.
+// 64-bit inputs encode to 1-9 bytes.
+//
+// 0x0000'0000'0000'0000 - 0x0000'0000'0000'007f : 0b0_______ 1 byte
+// 0x0000'0000'0000'0080 - 0x0000'0000'0000'3fff : 0b10______ 2 bytes
+// 0x0000'0000'0000'4000 - 0x0000'0000'001f'ffff : 0b110_____ 3 bytes
+// 0x0000'0000'0020'0000 - 0x0000'0000'0fff'ffff : 0b1110____ 4 bytes
+// 0x0000'0000'1000'0000 - 0x0000'0007'ffff'ffff : 0b11110___ 5 bytes
+// 0x0000'0008'0000'0000 - 0x0000'03ff'ffff'ffff : 0b111110__ 6 bytes
+// 0x0000'0400'0000'0000 - 0x0001'ffff'ffff'ffff : 0b1111110_ 7 bytes
+// 0x0002'0000'0000'0000 - 0x00ff'ffff'ffff'ffff : 0b11111110 8 bytes
+// 0x0100'0000'0000'0000 - 0xffff'ffff'ffff'ffff : 0b11111111 9 bytes
+//
+// Encoding Examples
+// -42 => ZigZag => 0x53 => 0x53
+// 42 => ZigZag => 0x54 => 0x54
+// 0x1 => 0x01
+// 0x12 => 0x12
+// 0x123 => 0x81 0x23
+// 0x1234 => 0x92 0x34
+// 0x12345 => 0xc1 0x23 0x45
+// 0x123456 => 0xd2 0x34 0x56
+// 0x1234567 => 0xe1 0x23 0x45 0x67
+// 0x12345678 => 0xf0 0x12 0x34 0x56 0x78
+// 0x123456789 => 0xf1 0x23 0x45 0x67 0x89
+// 0x123456789a => 0xf8 0x12 0x34 0x56 0x78 0x9a
+// 0x123456789ab => 0xfb 0x23 0x45 0x67 0x89 0xab
+// 0x123456789abc => 0xfc 0x12 0x34 0x56 0x78 0x9a 0xbc
+// 0x123456789abcd => 0xfd 0x23 0x45 0x67 0x89 0xab 0xcd
+// 0x123456789abcde => 0xfe 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde
+// 0x123456789abcdef => 0xff 0x01 0x23 0x45 0x67 0x89 0xab 0xcd 0xef
+// 0x123456789abcdef0 => 0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0
+
+/**
+ * Measure the length in bytes (1-9) of an encoded variable-length integer.
+ *
+ * @param InData A variable-length encoding of an (signed or unsigned) integer.
+ * @return The number of bytes used to encode the integer, in the range 1-9.
+ */
+inline uint32_t
+MeasureVarUInt(const void* InData)
+{
+ return CountLeadingZeros(uint8_t(~*static_cast<const uint8_t*>(InData))) - 23;
+}
+
+/** Measure the length in bytes (1-9) of an encoded variable-length integer. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(const void* InData)
+{
+ return MeasureVarUInt(InData);
+}
+
+/** Measure the number of bytes (1-5) required to encode the 32-bit input. */
+inline uint32_t
+MeasureVarUInt(uint32_t InValue)
+{
+ return uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1);
+}
+
+/** Measure the number of bytes (1-9) required to encode the 64-bit input. */
+inline uint32_t
+MeasureVarUInt(uint64_t InValue)
+{
+ return uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9));
+}
+
+/** Measure the number of bytes (1-5) required to encode the 32-bit input. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(int32_t InValue)
+{
+ return MeasureVarUInt(uint32_t((InValue >> 31) ^ (InValue << 1)));
+}
+
+/** Measure the number of bytes (1-9) required to encode the 64-bit input. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(int64_t InValue)
+{
+ return MeasureVarUInt(uint64_t((InValue >> 63) ^ (InValue << 1)));
+}
+
+/**
+ * Read a variable-length unsigned integer.
+ *
+ * @param InData A variable-length encoding of an unsigned integer.
+ * @param OutByteCount The number of bytes consumed from the input.
+ * @return An unsigned integer.
+ */
+inline uint64_t
+ReadVarUInt(const void* InData, uint32_t& OutByteCount)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InData);
+ OutByteCount = ByteCount;
+
+ const uint8_t* InBytes = static_cast<const uint8_t*>(InData);
+ uint64_t Value = *InBytes++ & uint8_t(0xff >> ByteCount);
+ switch (ByteCount - 1)
+ {
+ case 8:
+ Value <<= 8;
+ Value |= *InBytes++;
+ [[fallthrough]];
+ case 7:
+ Value <<= 8;
+ Value |= *InBytes++;
+ [[fallthrough]];
+ case 6:
+ Value <<= 8;
+ Value |= *InBytes++;
+ [[fallthrough]];
+ case 5:
+ Value <<= 8;
+ Value |= *InBytes++;
+ [[fallthrough]];
+ case 4:
+ Value <<= 8;
+ Value |= *InBytes++;
+ [[fallthrough]];
+ case 3:
+ Value <<= 8;
+ Value |= *InBytes++;
+ [[fallthrough]];
+ case 2:
+ Value <<= 8;
+ Value |= *InBytes++;
+ [[fallthrough]];
+ case 1:
+ Value <<= 8;
+ Value |= *InBytes++;
+ [[fallthrough]];
+ default:
+ return Value;
+ }
+}
+
+/**
+ * Read a variable-length signed integer.
+ *
+ * @param InData A variable-length encoding of a signed integer.
+ * @param OutByteCount The number of bytes consumed from the input.
+ * @return A signed integer.
+ */
+inline int64_t
+ReadVarInt(const void* InData, uint32_t& OutByteCount)
+{
+ const uint64_t Value = ReadVarUInt(InData, OutByteCount);
+ return -int64_t(Value & 1) ^ int64_t(Value >> 1);
+}
+
+/**
+ * Write a variable-length unsigned integer.
+ *
+ * @param InValue An unsigned integer to encode.
+ * @param OutData A buffer of at least 5 bytes to write the output to.
+ * @return The number of bytes used in the output.
+ */
+inline uint32_t
+WriteVarUInt(uint32_t InValue, void* OutData)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InValue);
+ uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1;
+ switch (ByteCount - 1)
+ {
+ case 4:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 3:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 2:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 1:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ default:
+ break;
+ }
+ *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue);
+ return ByteCount;
+}
+
+/**
+ * Write a variable-length unsigned integer.
+ *
+ * @param InValue An unsigned integer to encode.
+ * @param OutData A buffer of at least 9 bytes to write the output to.
+ * @return The number of bytes used in the output.
+ */
+inline uint32_t
+WriteVarUInt(uint64_t InValue, void* OutData)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InValue);
+ uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1;
+ switch (ByteCount - 1)
+ {
+ case 8:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 7:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 6:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 5:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 4:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 3:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 2:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ case 1:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ [[fallthrough]];
+ default:
+ break;
+ }
+ *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue);
+ return ByteCount;
+}
+
+/** Write a variable-length signed integer. \see \ref WriteVarUInt */
+inline uint32_t
+WriteVarInt(int32_t InValue, void* OutData)
+{
+ const uint32_t Value = uint32_t((InValue >> 31) ^ (InValue << 1));
+ return WriteVarUInt(Value, OutData);
+}
+
+/** Write a variable-length signed integer. \see \ref WriteVarUInt */
+inline uint32_t
+WriteVarInt(int64_t InValue, void* OutData)
+{
+ const uint64_t Value = uint64_t((InValue >> 63) ^ (InValue << 1));
+ return WriteVarUInt(Value, OutData);
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/windows.h b/src/zencore/include/zencore/windows.h
new file mode 100644
index 000000000..91828f0ec
--- /dev/null
+++ b/src/zencore/include/zencore/windows.h
@@ -0,0 +1,25 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+
+struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive-
+#ifndef NOMINMAX
+# define NOMINMAX // We don't want your min/max macros
+#endif
+#ifndef NOGDI
+# define NOGDI // We don't want your GetObject define
+#endif
+#ifndef WIN32_LEAN_AND_MEAN
+# define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef _WIN32_WINNT
+# define _WIN32_WINNT 0x0A00
+#endif
+#include <windows.h>
+#undef GetObject
+
+ZEN_THIRD_PARTY_INCLUDES_END
diff --git a/src/zencore/include/zencore/workthreadpool.h b/src/zencore/include/zencore/workthreadpool.h
new file mode 100644
index 000000000..0ddc65298
--- /dev/null
+++ b/src/zencore/include/zencore/workthreadpool.h
@@ -0,0 +1,48 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/blockingqueue.h>
+#include <zencore/refcount.h>
+
+#include <exception>
+#include <functional>
+#include <system_error>
+#include <thread>
+#include <vector>
+
+namespace zen {
+
+struct IWork : public RefCounted
+{
+ virtual void Execute() = 0;
+
+ inline std::exception_ptr GetException() { return m_Exception; }
+
+private:
+ std::exception_ptr m_Exception;
+
+ friend class WorkerThreadPool;
+};
+
+class WorkerThreadPool
+{
+public:
+ WorkerThreadPool(int InThreadCount);
+ ~WorkerThreadPool();
+
+ void ScheduleWork(Ref<IWork> Work);
+ void ScheduleWork(std::function<void()>&& Work);
+
+ [[nodiscard]] size_t PendingWork() const;
+
+private:
+ void WorkerThreadFunction();
+
+ std::vector<std::thread> m_WorkerThreads;
+ BlockingQueue<Ref<IWork>> m_WorkQueue;
+};
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/xxhash.h b/src/zencore/include/zencore/xxhash.h
new file mode 100644
index 000000000..04872f4c3
--- /dev/null
+++ b/src/zencore/include/zencore/xxhash.h
@@ -0,0 +1,89 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/memory.h>
+
+#include <xxh3.h>
+
+#include <compare>
+#include <string_view>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/**
+ * XXH3 hash
+ */
+struct XXH3_128
+{
+ uint8_t Hash[16];
+
+ static XXH3_128 MakeFrom(const void* data /* 16 bytes */)
+ {
+ XXH3_128 Xx;
+ memcpy(Xx.Hash, data, sizeof Xx);
+ return Xx;
+ }
+
+ static inline XXH3_128 HashMemory(const void* data, size_t byteCount)
+ {
+ XXH3_128 Hash;
+ XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits(data, byteCount));
+ return Hash;
+ }
+ static XXH3_128 HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); }
+ static XXH3_128 FromHexString(const char* string);
+ static XXH3_128 FromHexString(const std::string_view string);
+ const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static const int StringLength = 32;
+ typedef char String_t[StringLength + 1];
+
+ static XXH3_128 Zero; // Initialized to all zeros
+
+ inline auto operator<=>(const XXH3_128& rhs) const = default;
+
+ struct Hasher
+ {
+ size_t operator()(const XXH3_128& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct XXH3_128Stream
+{
+ /// Begin streaming hash compute (not needed on freshly constructed instance)
+ void Reset() { memset(&m_State, 0, sizeof m_State); }
+
+ /// Append another chunk
+ XXH3_128Stream& Append(const void* Data, size_t ByteCount)
+ {
+ XXH3_128bits_update(&m_State, Data, ByteCount);
+ return *this;
+ }
+
+ /// Append another chunk
+ XXH3_128Stream& Append(MemoryView Data) { return Append(Data.GetData(), Data.GetSize()); }
+
+ /// Obtain final hash. If you wish to reuse the instance call reset()
+ XXH3_128 GetHash()
+ {
+ XXH3_128 Hash;
+ XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits_digest(&m_State));
+ return Hash;
+ }
+
+private:
+ XXH3_state_s m_State{};
+};
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/zencore.h b/src/zencore/include/zencore/zencore.h
new file mode 100644
index 000000000..5bcd77239
--- /dev/null
+++ b/src/zencore/include/zencore/zencore.h
@@ -0,0 +1,383 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <cinttypes>
+#include <stdexcept>
+#include <string>
+#include <version>
+
+#ifndef ZEN_WITH_TESTS
+# define ZEN_WITH_TESTS 1
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+// Platform
+//
+
+#define ZEN_PLATFORM_WINDOWS 0
+#define ZEN_PLATFORM_LINUX 0
+#define ZEN_PLATFORM_MAC 0
+
+#ifdef _WIN32
+# undef ZEN_PLATFORM_WINDOWS
+# define ZEN_PLATFORM_WINDOWS 1
+#elif defined(__linux__)
+# undef ZEN_PLATFORM_LINUX
+# define ZEN_PLATFORM_LINUX 1
+#elif defined(__APPLE__)
+# undef ZEN_PLATFORM_MAC
+# define ZEN_PLATFORM_MAC 1
+#endif
+
+#if ZEN_PLATFORM_WINDOWS
+# if !defined(NOMINMAX)
+# define NOMINMAX // stops Windows.h from defining 'min/max' macros
+# endif
+# if !defined(NOGDI)
+# define NOGDI
+# endif
+# if !defined(WIN32_LEAN_AND_MEAN)
+# define WIN32_LEAN_AND_MEAN // cut-down what Windows.h defines
+# endif
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+// Compiler
+//
+
+#define ZEN_COMPILER_CLANG 0
+#define ZEN_COMPILER_MSC 0
+#define ZEN_COMPILER_GCC 0
+
+// Clang can define __GNUC__ and/or _MSC_VER so we check for Clang first
+#ifdef __clang__
+# undef ZEN_COMPILER_CLANG
+# define ZEN_COMPILER_CLANG 1
+#elif defined(_MSC_VER)
+# undef ZEN_COMPILER_MSC
+# define ZEN_COMPILER_MSC 1
+#elif defined(__GNUC__)
+# undef ZEN_COMPILER_GCC
+# define ZEN_COMPILER_GCC 1
+#else
+# error Unknown compiler
+#endif
+
+#if ZEN_COMPILER_MSC
+# pragma warning(disable : 4324) // warning C4324: '<type>': structure was padded due to alignment specifier
+# pragma warning(default : 4668) // warning C4668: 'symbol' is not defined as a preprocessor macro, replacing with '0' for 'directives'
+# pragma warning(default : 4100) // warning C4100: 'identifier' : unreferenced formal parameter
+#endif
+
+#ifndef ZEN_THIRD_PARTY_INCLUDES_START
+# if ZEN_COMPILER_MSC
+# define ZEN_THIRD_PARTY_INCLUDES_START \
+ __pragma(warning(push)) __pragma(warning(disable : 4668)) /* C4668: use of undefined preprocessor macro */ \
+ __pragma(warning(disable : 4305)) /* C4305: 'if': truncation from 'uint32' to 'bool' */ \
+ __pragma(warning(disable : 4267)) /* C4267: '=': conversion from 'size_t' to 'US' */ \
+ __pragma(warning(disable : 4127)) /* C4127: conditional expression is constant */ \
+ __pragma(warning(disable : 4189)) /* C4189: local variable is initialized but not referenced */
+# elif ZEN_COMPILER_CLANG
+# define ZEN_THIRD_PARTY_INCLUDES_START \
+ _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wundef\"") \
+ _Pragma("clang diagnostic ignored \"-Wunused-parameter\"") _Pragma("clang diagnostic ignored \"-Wunused-variable\"")
+# elif ZEN_COMPILER_GCC
+# define ZEN_THIRD_PARTY_INCLUDES_START \
+ _Pragma("GCC diagnostic push") /* NB. ignoring -Wundef doesn't work with GCC */ \
+ _Pragma("GCC diagnostic ignored \"-Wunused-parameter\"") _Pragma("GCC diagnostic ignored \"-Wunused-variable\"")
+# endif
+#endif
+
+#ifndef ZEN_THIRD_PARTY_INCLUDES_END
+# if ZEN_COMPILER_MSC
+# define ZEN_THIRD_PARTY_INCLUDES_END __pragma(warning(pop))
+# elif ZEN_COMPILER_CLANG
+# define ZEN_THIRD_PARTY_INCLUDES_END _Pragma("clang diagnostic pop")
+# elif ZEN_COMPILER_GCC
+# define ZEN_THIRD_PARTY_INCLUDES_END _Pragma("GCC diagnostic pop")
+# endif
+#endif
+
+#if ZEN_COMPILER_MSC
+# define ZEN_DEBUG_BREAK() \
+ do \
+ { \
+ __debugbreak(); \
+ } while (0)
+#else
+# define ZEN_DEBUG_BREAK() \
+ do \
+ { \
+ __builtin_trap(); \
+ } while (0)
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+// C++20 support
+//
+
+// Clang
+#if ZEN_COMPILER_CLANG && __clang_major__ < 12
+# error clang-12 onwards is required for C++20 support
+#endif
+
+// GCC
+#if ZEN_COMPILER_GCC && __GNUC__ < 11
+# error GCC-11 onwards is required for C++20 support
+#endif
+
+// GNU libstdc++
+#if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11
+# error GNU libstdc++-11 onwards is required for C++20 support
+#endif
+
+// LLVM libc++
+#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 12000
+# error LLVM libc++-12 onwards is required for C++20 support
+#endif
+
+// At the time of writing only ver >= 13 of LLVM's libc++ has an implementation
+// of std::integral. Some platforms like Ubuntu and Mac OS are still on 12.
+#if defined(__cpp_lib_concepts)
+# include <concepts>
+template<class T>
+concept Integral = std::integral<T>;
+template<class T>
+concept SignedIntegral = std::signed_integral<T>;
+template<class T>
+concept UnsignedIntegral = std::unsigned_integral<T>;
+template<class F, class... A>
+concept Invocable = std::invocable<F, A...>;
+template<class D, class B>
+concept DerivedFrom = std::derived_from<D, B>;
+#else
+template<class T>
+concept Integral = std::is_integral_v<T>;
+template<class T>
+concept SignedIntegral = Integral<T> && std::is_signed_v<T>;
+template<class T>
+concept UnsignedIntegral = Integral<T> && !std::is_signed_v<T>;
+template<class F, class... A>
+concept Invocable = requires(F&& f, A&&... a)
+{
+ std::invoke(std::forward<F>(f), std::forward<A>(a)...);
+};
+template<class D, class B>
+concept DerivedFrom = std::is_base_of_v<B, D> && std::is_convertible_v<const volatile D*, const volatile B*>;
+#endif
+
+#if defined(__cpp_lib_ranges)
+template<typename T>
+concept ContiguousRange = std::ranges::contiguous_range<T>;
+#else
+template<typename T>
+concept ContiguousRange = true;
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+// Architecture
+//
+
+#if defined(__amd64__) || defined(_M_X64)
+# define ZEN_ARCH_X64 1
+# define ZEN_ARCH_ARM64 0
+#elif defined(__arm64__) || defined(_M_ARM64)
+# define ZEN_ARCH_X64 0
+# define ZEN_ARCH_ARM64 1
+#else
+# error Unknown architecture
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+// Build flavor
+//
+
+#ifdef NDEBUG
+# define ZEN_BUILD_DEBUG 0
+# define ZEN_BUILD_RELEASE 1
+#else
+# define ZEN_BUILD_DEBUG 1
+# define ZEN_BUILD_RELEASE 0
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+#define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1
+
+#if defined(__SIZEOF_WCHAR_T__) && __SIZEOF_WCHAR_T__ == 4
+# define ZEN_SIZEOF_WCHAR_T 4
+#else
+static_assert(sizeof(wchar_t) == 2, "wchar_t is expected to be two bytes in size");
+# define ZEN_SIZEOF_WCHAR_T 2
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+// Assert
+//
+
+#if ZEN_PLATFORM_WINDOWS
+// Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable.
+# define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name))
+# define ZEN_FORCENOINLINE __declspec(noinline) /* Force code to NOT be inline */
+# define LINE_TERMINATOR_ANSI "\r\n"
+#else
+# define ZEN_CODE_SECTION(Name)
+# define ZEN_FORCENOINLINE
+# define LINE_TERMINATOR_ANSI "\n"
+#endif
+
+#if ZEN_ARCH_ARM64
+// On ARM we can't do this because the executable will require jumps larger
+// than the branch instruction can handle. Clang will only generate
+// the trampolines in the .text segment of the binary. If the zcold segment
+// is present it will generate code that it cannot link.
+# define ZEN_DEBUG_SECTION
+#else
+// We'll put all assert implementation code into a separate section in the linked
+// executable. This code should never execute so using a separate section keeps
+// it well off the hot path and hopefully out of the instruction cache. It also
+// facilitates reasoning about the makeup of a compiled/linked binary.
+# define ZEN_DEBUG_SECTION ZEN_CODE_SECTION(".zcold")
+#endif
+
+namespace zen
+{
+ class AssertException : public std::logic_error
+ {
+ public:
+ AssertException(const char* Msg) : std::logic_error(Msg) {}
+ };
+
+ struct AssertImpl
+ {
+ static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecAssert
+ [[noreturn]] (const char* Filename, int LineNumber, const char* FunctionName, const char* Msg)
+ {
+ CurrentAssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg);
+ throw AssertException{Msg};
+ }
+
+ protected:
+ virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION OnAssert(const char* Filename,
+ int LineNumber,
+ const char* FunctionName,
+ const char* Msg)
+ {
+ (void(Filename));
+ (void(LineNumber));
+ (void(FunctionName));
+ (void(Msg));
+ }
+ static AssertImpl DefaultAssertImpl;
+ static AssertImpl* CurrentAssertImpl;
+ };
+
+} // namespace zen
+
+#define ZEN_ASSERT(x, ...) \
+ do \
+ { \
+ if (x) [[unlikely]] \
+ break; \
+ zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \
+ } while (false)
+
+#ifndef NDEBUG
+# define ZEN_ASSERT_SLOW(x, ...) \
+ do \
+ { \
+ if (x) [[unlikely]] \
+ break; \
+ zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \
+ } while (false)
+#else
+# define ZEN_ASSERT_SLOW(x, ...)
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+#ifdef __clang__
+template<typename T>
+auto ZenArrayCountHelper(T& t) -> typename std::enable_if<__is_array(T), char (&)[sizeof(t) / sizeof(t[0]) + 1]>::type;
+#else
+template<typename T, uint32_t N>
+char (&ZenArrayCountHelper(const T (&)[N]))[N + 1];
+#endif
+
+#define ZEN_ARRAY_COUNT(array) (sizeof(ZenArrayCountHelper(array)) - 1)
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_COMPILER_MSC
+# define ZEN_NOINLINE __declspec(noinline)
+#else
+# define ZEN_NOINLINE __attribute__((noinline))
+#endif
+
+#if ZEN_PLATFORM_WINDOWS
+# define ZEN_EXE_SUFFIX_LITERAL ".exe"
+#else
+# define ZEN_EXE_SUFFIX_LITERAL
+#endif
+
+#define ZEN_UNUSED(...) ((void)__VA_ARGS__)
+#define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__)
+#define ZENCORE_API // Placeholder to allow DLL configs in the future (maybe)
+
+namespace zen {
+
+ZENCORE_API bool IsApplicationExitRequested();
+ZENCORE_API void RequestApplicationExit(int ExitCode);
+ZENCORE_API bool IsDebuggerPresent();
+ZENCORE_API void SetIsInteractiveSession(bool Value);
+ZENCORE_API bool IsInteractiveSession();
+
+ZENCORE_API void zencore_forcelinktests();
+
+} // namespace zen
+
+//////////////////////////////////////////////////////////////////////////
+
+#ifndef ZEN_USE_MIMALLOC
+# if ZEN_ARCH_ARM64
+ // The vcpkg mimalloc port doesn't support Arm targets
+# define ZEN_USE_MIMALLOC 0
+# else
+# define ZEN_USE_MIMALLOC 1
+# endif
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_COMPILER_MSC
+# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off))
+# define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on))
+#elif ZEN_COMPILER_GCC
+# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("GCC push_options") _Pragma("GCC optimize (\"O0\")")
+# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("GCC pop_options")
+#elif ZEN_COMPILER_CLANG
+# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize off")
+# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize on")
+#endif
+
+// Set up optimization control macros, now that we have both the build settings and the platform macros
+#define ZEN_DISABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL
+
+#if ZEN_BUILD_DEBUG
+# define ZEN_ENABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL
+#else
+# define ZEN_ENABLE_OPTIMIZATION ZEN_ENABLE_OPTIMIZATION_ACTUAL
+#endif
+
+#define ZEN_ENABLE_OPTIMIZATION_ALWAYS ZEN_ENABLE_OPTIMIZATION_ACTUAL
+
+//////////////////////////////////////////////////////////////////////////
+
+#ifndef ZEN_WITH_TRACE
+# define ZEN_WITH_TRACE 0
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+using ThreadId_t = uint32_t;