aboutsummaryrefslogtreecommitdiff
path: root/zencore/include
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2021-05-11 13:05:39 +0200
committerStefan Boberg <[email protected]>2021-05-11 13:05:39 +0200
commitf8d9ac5d13dd37b8b57af0478e77ba1e75c813aa (patch)
tree1daf7621e110d48acd5e12e3073ce48ef0dd11b2 /zencore/include
downloadzen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.tar.xz
zen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.zip
Adding zenservice code
Diffstat (limited to 'zencore/include')
-rw-r--r--zencore/include/zencore/atomic.h43
-rw-r--r--zencore/include/zencore/blake3.h57
-rw-r--r--zencore/include/zencore/compactbinary.h1335
-rw-r--r--zencore/include/zencore/compactbinarybuilder.h633
-rw-r--r--zencore/include/zencore/compactbinarypackage.h305
-rw-r--r--zencore/include/zencore/compactbinaryvalidation.h192
-rw-r--r--zencore/include/zencore/compress.h53
-rw-r--r--zencore/include/zencore/endian.h61
-rw-r--r--zencore/include/zencore/enumflags.h61
-rw-r--r--zencore/include/zencore/except.h60
-rw-r--r--zencore/include/zencore/filesystem.h74
-rw-r--r--zencore/include/zencore/fmtutils.h49
-rw-r--r--zencore/include/zencore/httpclient.h18
-rw-r--r--zencore/include/zencore/httpserver.h373
-rw-r--r--zencore/include/zencore/intmath.h140
-rw-r--r--zencore/include/zencore/iobuffer.h272
-rw-r--r--zencore/include/zencore/iohash.h95
-rw-r--r--zencore/include/zencore/md5.h50
-rw-r--r--zencore/include/zencore/memory.h213
-rw-r--r--zencore/include/zencore/meta.h30
-rw-r--r--zencore/include/zencore/refcount.h144
-rw-r--r--zencore/include/zencore/scopeguard.h33
-rw-r--r--zencore/include/zencore/sha1.h76
-rw-r--r--zencore/include/zencore/sharedbuffer.h169
-rw-r--r--zencore/include/zencore/snapshot_manifest.h57
-rw-r--r--zencore/include/zencore/stats.h66
-rw-r--r--zencore/include/zencore/stream.h318
-rw-r--r--zencore/include/zencore/streamutil.h118
-rw-r--r--zencore/include/zencore/string.h595
-rw-r--r--zencore/include/zencore/thread.h118
-rw-r--r--zencore/include/zencore/timer.h41
-rw-r--r--zencore/include/zencore/trace.h91
-rw-r--r--zencore/include/zencore/uid.h78
-rw-r--r--zencore/include/zencore/varint.h255
-rw-r--r--zencore/include/zencore/windows.h10
-rw-r--r--zencore/include/zencore/xxhash.h87
-rw-r--r--zencore/include/zencore/zencore.h134
37 files changed, 6504 insertions, 0 deletions
diff --git a/zencore/include/zencore/atomic.h b/zencore/include/zencore/atomic.h
new file mode 100644
index 000000000..457128bd4
--- /dev/null
+++ b/zencore/include/zencore/atomic.h
@@ -0,0 +1,43 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <intrin.h>
+#include <cinttypes>
+
+namespace zen {
+
+inline uint32_t
+AtomicIncrement(volatile uint32_t& value)
+{
+ return _InterlockedIncrement((long volatile*)&value);
+}
+inline uint32_t
+AtomicDecrement(volatile uint32_t& value)
+{
+ return _InterlockedDecrement((long volatile*)&value);
+}
+
+inline uint64_t
+AtomicIncrement(volatile uint64_t& value)
+{
+ return _InterlockedIncrement64((__int64 volatile*)&value);
+}
+inline uint64_t
+AtomicDecrement(volatile uint64_t& value)
+{
+ return _InterlockedDecrement64((__int64 volatile*)&value);
+}
+
+inline uint32_t
+AtomicAdd(volatile uint32_t& value, uint32_t amount)
+{
+ return _InterlockedExchangeAdd((long volatile*)&value, amount);
+}
+inline uint64_t
+AtomicAdd(volatile uint64_t& value, uint64_t amount)
+{
+ return _InterlockedExchangeAdd64((__int64 volatile*)&value, amount);
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/blake3.h b/zencore/include/zencore/blake3.h
new file mode 100644
index 000000000..1ef921c30
--- /dev/null
+++ b/zencore/include/zencore/blake3.h
@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <cinttypes>
+#include <compare>
+#include <cstring>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/**
+ * BLAKE3 hash - 256 bits
+ */
+struct BLAKE3
+{
+ uint8_t Hash[32];
+
+ inline auto operator<=>(const BLAKE3& rhs) const = default;
+
+ static BLAKE3 HashMemory(const void* data, size_t byteCount);
+ static BLAKE3 FromHexString(const char* string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static const int StringLength = 64;
+ typedef char String_t[StringLength + 1];
+
+ static BLAKE3 Zero; // Initialized to all zeroes
+
+ struct Hasher
+ {
+ size_t operator()(const BLAKE3& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct BLAKE3Stream
+{
+ BLAKE3Stream();
+
+ void Reset(); /// Begin streaming hash compute (not needed on freshly constructed instance)
+ BLAKE3Stream& Append(const void* data, size_t byteCount); /// Append another chunk
+ BLAKE3 GetHash(); /// Obtain final hash. If you wish to reuse the instance call reset()
+
+private:
+ alignas(16) uint8_t m_HashState[2048];
+};
+
+void blake3_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
new file mode 100644
index 000000000..c2d276c21
--- /dev/null
+++ b/zencore/include/zencore/compactbinary.h
@@ -0,0 +1,1335 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/enumflags.h>
+#include <zencore/intmath.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/memory.h>
+#include <zencore/meta.h>
+#include <zencore/sharedbuffer.h>
+#include <zencore/uid.h>
+#include <zencore/varint.h>
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class CbObjectView;
+class CbArrayView;
+class BinaryReader;
+class BinaryWriter;
+
+class DateTime
+{
+public:
+ explicit DateTime(uint64_t InTicks) : Ticks(InTicks) {}
+ inline DateTime(int Year, int Month, int Day, int Hours = 0, int Minutes = 0, int Seconds = 0, int MilliSeconds = 0)
+ {
+ Set(Year, Month, Day, Hours, Minutes, Seconds, MilliSeconds);
+ }
+
+ inline uint64_t GetTicks() const { return Ticks; }
+ inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
+ inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+private:
+ void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond);
+ uint64_t Ticks;
+};
+
+class TimeSpan
+{
+public:
+ explicit TimeSpan(uint64_t InTicks) : Ticks(InTicks) {}
+ inline TimeSpan(int Hours, int Minutes, int Seconds) { Set(0, Hours, Minutes, Seconds, 0); }
+ inline TimeSpan(int Days, int Hours, int Minutes, int Seconds) { Set(Days, Hours, Minutes, Seconds, 0); }
+ inline TimeSpan(int Days, int Hours, int Minutes, int Seconds, int Nanos) { Set(Days, Hours, Minutes, Seconds, Nanos); }
+
+ inline uint64_t GetTicks() const { return Ticks; }
+ inline bool operator==(const TimeSpan& Rhs) const { return Ticks == Rhs.Ticks; }
+ inline auto operator<=>(const TimeSpan& Rhs) const { return Ticks - Rhs.Ticks; }
+
+ /**
+ * Time span related constants.
+ */
+
+ /** The maximum number of ticks that can be represented in FTimespan. */
+ static constexpr int64_t MaxTicks = 9223372036854775807;
+
+ /** The minimum number of ticks that can be represented in FTimespan. */
+ static constexpr int64_t MinTicks = -9223372036854775807 - 1;
+
+ /** The number of nanoseconds per tick. */
+ static constexpr int64_t NanosecondsPerTick = 100;
+
+ /** The number of timespan ticks per day. */
+ static constexpr int64_t TicksPerDay = 864000000000;
+
+ /** The number of timespan ticks per hour. */
+ static constexpr int64_t TicksPerHour = 36000000000;
+
+ /** The number of timespan ticks per microsecond. */
+ static constexpr int64_t TicksPerMicrosecond = 10;
+
+ /** The number of timespan ticks per millisecond. */
+ static constexpr int64_t TicksPerMillisecond = 10000;
+
+ /** The number of timespan ticks per minute. */
+ static constexpr int64_t TicksPerMinute = 600000000;
+
+ /** The number of timespan ticks per second. */
+ static constexpr int64_t TicksPerSecond = 10000000;
+
+ /** The number of timespan ticks per week. */
+ static constexpr int64_t TicksPerWeek = 6048000000000;
+
+ /** The number of timespan ticks per year (365 days, not accounting for leap years). */
+ static constexpr int64_t TicksPerYear = 365 * TicksPerDay;
+
+private:
+ void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano);
+
+ uint64_t Ticks;
+};
+
+struct Guid
+{
+ uint32_t A, B, C, D;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+/**
+ * Field types and flags for CbField.
+ *
+ * This is a private type and is only declared here to enable inline use below.
+ *
+ * DO NOT CHANGE THE VALUE OF ANY MEMBERS OF THIS ENUM!
+ * BACKWARD COMPATIBILITY REQUIRES THAT THESE VALUES BE FIXED!
+ * SERIALIZATION USES HARD-CODED CONSTANTS BASED ON THESE VALUES!
+ */
+enum class CbFieldType : uint8_t
+{
+ /** A field type that does not occur in a valid object. */
+ None = 0x00,
+
+ /** Null. Payload is empty. */
+ Null = 0x01,
+
+ /**
+ * Object is an array of fields with unique non-empty names.
+ *
+ * Payload is a VarUInt byte count for the encoded fields followed by the fields.
+ */
+ Object = 0x02,
+ /**
+ * UniformObject is an array of fields with the same field types and unique non-empty names.
+ *
+ * Payload is a VarUInt byte count for the encoded fields followed by the fields.
+ */
+ UniformObject = 0x03,
+
+ /**
+ * Array is an array of fields with no name that may be of different types.
+ *
+ * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by the fields.
+ */
+ Array = 0x04,
+ /**
+ * UniformArray is an array of fields with no name and with the same field type.
+ *
+ * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by field type,
+ * followed by the fields without their field type.
+ */
+ UniformArray = 0x05,
+
+ /** Binary. Payload is a VarUInt byte count followed by the data. */
+ Binary = 0x06,
+
+ /** String in UTF-8. Payload is a VarUInt byte count then an unterminated UTF-8 string. */
+ String = 0x07,
+
+ /**
+ * Non-negative integer with the range of a 64-bit unsigned integer.
+ *
+ * Payload is the value encoded as a VarUInt.
+ */
+ IntegerPositive = 0x08,
+ /**
+ * Negative integer with the range of a 64-bit signed integer.
+ *
+ * Payload is the ones' complement of the value encoded as a VarUInt.
+ */
+ IntegerNegative = 0x09,
+
+ /** Single precision float. Payload is one big endian IEEE 754 binary32 float. */
+ Float32 = 0x0a,
+ /** Double precision float. Payload is one big endian IEEE 754 binary64 float. */
+ Float64 = 0x0b,
+
+ /** Boolean false value. Payload is empty. */
+ BoolFalse = 0x0c,
+ /** Boolean true value. Payload is empty. */
+ BoolTrue = 0x0d,
+
+ /**
+ * CompactBinaryAttachment is a reference to a compact binary attachment stored externally.
+ *
+ * Payload is a 160-bit hash digest of the referenced compact binary data.
+ */
+ CompactBinaryAttachment = 0x0e,
+ /**
+ * BinaryAttachment is a reference to a binary attachment stored externally.
+ *
+ * Payload is a 160-bit hash digest of the referenced binary data.
+ */
+ BinaryAttachment = 0x0f,
+
+ /** Hash. Payload is a 160-bit hash digest. */
+ Hash = 0x10,
+ /** UUID/GUID. Payload is a 128-bit UUID as defined by RFC 4122. */
+ Uuid = 0x11,
+
+ /**
+ * Date and time between 0001-01-01 00:00:00.0000000 and 9999-12-31 23:59:59.9999999.
+ *
+ * Payload is a big endian int64 count of 100ns ticks since 0001-01-01 00:00:00.0000000.
+ */
+ DateTime = 0x12,
+ /**
+ * Difference between two date/time values.
+ *
+ * Payload is a big endian int64 count of 100ns ticks in the span, and may be negative.
+ */
+ TimeSpan = 0x13,
+
+ /**
+ * Object ID
+ *
+ * Payload is a 12-byte opaque identifier
+ */
+ ObjectId = 0x14,
+
+ /**
+ * CustomById identifies the sub-type of its payload by an integer identifier.
+ *
+ * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed
+ * by a VarUInt of the sub-type identifier then the payload of the sub-type.
+ */
+ CustomById = 0x1e,
+ /**
+ * CustomByType identifies the sub-type of its payload by a string identifier.
+ *
+ * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed
+ * by a VarUInt byte count of the unterminated sub-type identifier, then the sub-type identifier
+ * without termination, then the payload of the sub-type.
+ */
+ CustomByName = 0x1f,
+
+ /** Reserved for future use as a flag. Do not add types in this range. */
+ Reserved = 0x20,
+
+ /**
+ * A transient flag which indicates that the object or array containing this field has stored
+ * the field type before the payload and name. Non-uniform objects and fields will set this.
+ *
+ * Note: Since the flag must never be serialized, this bit may be repurposed in the future.
+ */
+ HasFieldType = 0x40,
+
+ /** A persisted flag which indicates that the field has a name stored before the payload. */
+ HasFieldName = 0x80,
+};
+
+ENUM_CLASS_FLAGS(CbFieldType);
+
+/** Functions that operate on CbFieldType. */
+class CbFieldTypeOps
+{
+ static constexpr CbFieldType SerializedTypeMask = CbFieldType(0b1011'1111);
+ static constexpr CbFieldType TypeMask = CbFieldType(0b0011'1111);
+ static constexpr CbFieldType ObjectMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType ObjectBase = CbFieldType(0b0000'0010);
+ static constexpr CbFieldType ArrayMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType ArrayBase = CbFieldType(0b0000'0100);
+ static constexpr CbFieldType IntegerMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType IntegerBase = CbFieldType(0b0000'1000);
+ static constexpr CbFieldType FloatMask = CbFieldType(0b0011'1100);
+ static constexpr CbFieldType FloatBase = CbFieldType(0b0000'1000);
+ static constexpr CbFieldType BoolMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType BoolBase = CbFieldType(0b0000'1100);
+ static constexpr CbFieldType AttachmentMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType AttachmentBase = CbFieldType(0b0000'1110);
+
+ static void StaticAssertTypeConstants();
+
+public:
+ /** The type with flags removed. */
+ static constexpr inline CbFieldType GetType(CbFieldType Type) { return Type & TypeMask; }
+ /** The type with transient flags removed. */
+ static constexpr inline CbFieldType GetSerializedType(CbFieldType Type) { return Type & SerializedTypeMask; }
+
+ static constexpr inline bool HasFieldType(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldType); }
+ static constexpr inline bool HasFieldName(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldName); }
+
+ static constexpr inline bool IsNone(CbFieldType Type) { return GetType(Type) == CbFieldType::None; }
+ static constexpr inline bool IsNull(CbFieldType Type) { return GetType(Type) == CbFieldType::Null; }
+
+ static constexpr inline bool IsObject(CbFieldType Type) { return (Type & ObjectMask) == ObjectBase; }
+ static constexpr inline bool IsArray(CbFieldType Type) { return (Type & ArrayMask) == ArrayBase; }
+
+ static constexpr inline bool IsBinary(CbFieldType Type) { return GetType(Type) == CbFieldType::Binary; }
+ static constexpr inline bool IsString(CbFieldType Type) { return GetType(Type) == CbFieldType::String; }
+
+ static constexpr inline bool IsInteger(CbFieldType Type) { return (Type & IntegerMask) == IntegerBase; }
+ /** Whether the field is a float, or integer due to implicit conversion. */
+ static constexpr inline bool IsFloat(CbFieldType Type) { return (Type & FloatMask) == FloatBase; }
+ static constexpr inline bool IsBool(CbFieldType Type) { return (Type & BoolMask) == BoolBase; }
+
+ static constexpr inline bool IsCompactBinaryAttachment(CbFieldType Type)
+ {
+ return GetType(Type) == CbFieldType::CompactBinaryAttachment;
+ }
+ static constexpr inline bool IsBinaryAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::BinaryAttachment; }
+ static constexpr inline bool IsAttachment(CbFieldType Type) { return (Type & AttachmentMask) == AttachmentBase; }
+
+ static constexpr inline bool IsHash(CbFieldType Type)
+ {
+ switch (GetType(Type))
+ {
+ case CbFieldType::Hash:
+ case CbFieldType::BinaryAttachment:
+ case CbFieldType::CompactBinaryAttachment:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; }
+ static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; }
+
+ static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; }
+ static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; }
+
+ /** Whether the type is or may contain fields of any attachment type. */
+ static constexpr inline bool MayContainAttachments(CbFieldType Type)
+ {
+ // The use of !! will suppress V792 from static analysis. Using //-V792 did not work.
+ return !!IsObject(Type) | !!IsArray(Type) | !!IsAttachment(Type);
+ }
+};
+
+/** Errors that can occur when accessing a field. */
+enum class CbFieldError : uint8_t
+{
+ /** The field is not in an error state. */
+ None,
+ /** The value type does not match the requested type. */
+ TypeError,
+ /** The value is out of range for the requested type. */
+ RangeError,
+};
+
+class ICbVisitor
+{
+public:
+ virtual void SetName(std::string_view Name) = 0;
+ virtual void BeginObject() = 0;
+ virtual void EndObject() = 0;
+ virtual void BeginArray() = 0;
+ virtual void EndArray() = 0;
+ virtual void VisitNull() = 0;
+ virtual void VisitBinary(SharedBuffer Value) = 0;
+ virtual void VisitString(std::string_view Value) = 0;
+ virtual void VisitInteger(int64_t Value) = 0;
+ virtual void VisitInteger(uint64_t Value) = 0;
+ virtual void VisitFloat(float Value) = 0;
+ virtual void VisitDouble(double Value) = 0;
+ virtual void VisitBool(bool value) = 0;
+ virtual void VisitCbAttachment(const IoHash& Value) = 0;
+ virtual void VisitBinaryAttachment(const IoHash& Value) = 0;
+ virtual void VisitHash(const IoHash& Value) = 0;
+ virtual void VisitUuid(const Guid& Value) = 0;
+ virtual void VisitObjectId(const Oid& Value) = 0;
+ virtual void VisitDateTime(DateTime Value) = 0;
+ virtual void VisitTimeSpan(TimeSpan Value) = 0;
+};
+
+/**
+ * An atom of data in the compact binary format.
+ *
+ * Accessing the value of a field is always a safe operation, even if accessed as the wrong type.
+ * An invalid access will return a default value for the requested type, and set an error code on
+ * the field that can be checked with GetLastError and HasLastError. A valid access will clear an
+ * error from a previous invalid access.
+ *
+ * A field is encoded in one or more bytes, depending on its type and the type of object or array
+ * that contains it. A field of an object or array which is non-uniform encodes its field type in
+ * the first byte, and includes the HasFieldName flag for a field in an object. The field name is
+ * encoded in a variable-length unsigned integer of its size in bytes, for named fields, followed
+ * by that many bytes of the UTF-8 encoding of the name with no null terminator. The remainder of
+ * the field is the payload and is described in the field type enum. Every field must be uniquely
+ * addressable when encoded, which means a zero-byte field is not permitted, and only arises in a
+ * uniform array of fields with no payload, where the answer is to encode as a non-uniform array.
+ *
+ * This type only provides a view into memory and does not perform any memory management itself.
+ * Use CbFieldRef to hold a reference to the underlying memory when necessary.
+ */
+
+class CbFieldView
+{
+public:
+ CbFieldView() = default;
+
+ ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType);
+
+ /** Returns the name of the field if it has a name, otherwise an empty view. */
+ constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); }
+
+ ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView());
+ ZENCORE_API CbObjectView AsObjectView();
+ ZENCORE_API CbArrayView AsArrayView();
+ ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view());
+
+ ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const;
+
+ /** Access the field as an int8. Returns the provided default on error. */
+ inline int8_t AsInt8(int8_t Default = 0) { return AsInteger<int8_t>(Default); }
+ /** Access the field as an int16. Returns the provided default on error. */
+ inline int16_t AsInt16(int16_t Default = 0) { return AsInteger<int16_t>(Default); }
+ /** Access the field as an int32. Returns the provided default on error. */
+ inline int32_t AsInt32(int32_t Default = 0) { return AsInteger<int32_t>(Default); }
+ /** Access the field as an int64. Returns the provided default on error. */
+ inline int64_t AsInt64(int64_t Default = 0) { return AsInteger<int64_t>(Default); }
+ /** Access the field as a uint8. Returns the provided default on error. */
+ inline uint8_t AsUInt8(uint8_t Default = 0) { return AsInteger<uint8_t>(Default); }
+ /** Access the field as a uint16. Returns the provided default on error. */
+ inline uint16_t AsUInt16(uint16_t Default = 0) { return AsInteger<uint16_t>(Default); }
+ /** Access the field as a uint32. Returns the provided default on error. */
+ inline uint32_t AsUInt32(uint32_t Default = 0) { return AsInteger<uint32_t>(Default); }
+ /** Access the field as a uint64. Returns the provided default on error. */
+ inline uint64_t AsUInt64(uint64_t Default = 0) { return AsInteger<uint64_t>(Default); }
+
+ /** Access the field as a float. Returns the provided default on error. */
+ ZENCORE_API float AsFloat(float Default = 0.0f);
+ /** Access the field as a double. Returns the provided default on error. */
+ ZENCORE_API double AsDouble(double Default = 0.0);
+
+ /** Access the field as a bool. Returns the provided default on error. */
+ ZENCORE_API bool AsBool(bool bDefault = false);
+
+ /** Access the field as a hash referencing a compact binary attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsCompactBinaryAttachment(const IoHash& Default = IoHash());
+ /** Access the field as a hash referencing a binary attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsBinaryAttachment(const IoHash& Default = IoHash());
+ /** Access the field as a hash referencing an attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsAttachment(const IoHash& Default = IoHash());
+
+ /** Access the field as a hash. Returns the provided default on error. */
+ ZENCORE_API IoHash AsHash(const IoHash& Default = IoHash());
+
+ /** Access the field as a UUID. Returns a nil UUID on error. */
+ ZENCORE_API Guid AsUuid();
+ /** Access the field as a UUID. Returns the provided default on error. */
+ ZENCORE_API Guid AsUuid(const Guid& Default);
+
+ /** Access the field as an OID. Returns a nil OID on error. */
+ ZENCORE_API Oid AsObjectId();
+ /** Access the field as a OID. Returns the provided default on error. */
+ ZENCORE_API Oid AsObjectId(const Oid& Default);
+
+ /** Access the field as a date/time tick count. Returns the provided default on error. */
+ ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0);
+
+ /** Access the field as a date/time. Returns a date/time at the epoch on error. */
+ ZENCORE_API DateTime AsDateTime();
+ /** Access the field as a date/time. Returns the provided default on error. */
+ ZENCORE_API DateTime AsDateTime(DateTime Default);
+
+ /** Access the field as a timespan tick count. Returns the provided default on error. */
+ ZENCORE_API int64_t AsTimeSpanTicks(int64_t Default = 0);
+
+ /** Access the field as a timespan. Returns an empty timespan on error. */
+ ZENCORE_API TimeSpan AsTimeSpan();
+ /** Access the field as a timespan. Returns the provided default on error. */
+ ZENCORE_API TimeSpan AsTimeSpan(TimeSpan Default);
+
+ /** True if the field has a name. */
+ constexpr inline bool HasName() const { return CbFieldTypeOps::HasFieldName(Type); }
+
+ constexpr inline bool IsNull() const { return CbFieldTypeOps::IsNull(Type); }
+
+ constexpr inline bool IsObject() const { return CbFieldTypeOps::IsObject(Type); }
+ constexpr inline bool IsArray() const { return CbFieldTypeOps::IsArray(Type); }
+
+ constexpr inline bool IsBinary() const { return CbFieldTypeOps::IsBinary(Type); }
+ constexpr inline bool IsString() const { return CbFieldTypeOps::IsString(Type); }
+
+ /** Whether the field is an integer of unspecified range and sign. */
+ constexpr inline bool IsInteger() const { return CbFieldTypeOps::IsInteger(Type); }
+ /** Whether the field is a float, or integer that supports implicit conversion. */
+ constexpr inline bool IsFloat() const { return CbFieldTypeOps::IsFloat(Type); }
+ constexpr inline bool IsBool() const { return CbFieldTypeOps::IsBool(Type); }
+
+ constexpr inline bool IsCompactBinaryAttachment() const { return CbFieldTypeOps::IsCompactBinaryAttachment(Type); }
+ constexpr inline bool IsBinaryAttachment() const { return CbFieldTypeOps::IsBinaryAttachment(Type); }
+ constexpr inline bool IsAttachment() const { return CbFieldTypeOps::IsAttachment(Type); }
+
+ constexpr inline bool IsHash() const { return CbFieldTypeOps::IsHash(Type); }
+ constexpr inline bool IsUuid() const { return CbFieldTypeOps::IsUuid(Type); }
+ constexpr inline bool IsObjectId() const { return CbFieldTypeOps::IsObjectId(Type); }
+
+ constexpr inline bool IsDateTime() const { return CbFieldTypeOps::IsDateTime(Type); }
+ constexpr inline bool IsTimeSpan() const { return CbFieldTypeOps::IsTimeSpan(Type); }
+
+ /** Whether the field has a value. */
+ constexpr inline explicit operator bool() const { return HasValue(); }
+
+ /**
+ * Whether the field has a value.
+ *
+ * All fields in a valid object or array have a value. A field with no value is returned when
+ * finding a field by name fails or when accessing an iterator past the end.
+ */
+ constexpr inline bool HasValue() const { return !CbFieldTypeOps::IsNone(Type); };
+
+ /** Whether the last field access encountered an error. */
+ constexpr inline bool HasError() const { return Error != CbFieldError::None; }
+
+ /** The type of error that occurred on the last field access, or None. */
+ constexpr inline CbFieldError GetError() const { return Error; }
+
+ /** Returns the size of the field in bytes, including the type and name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the field, including the type and name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& HashStream) const;
+
+ /** Feed the field (including type and name) to the stream function */
+ inline void WriteToStream(auto Hash) const
+ {
+ const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type);
+ Hash(&SerializedType, sizeof(SerializedType));
+ auto View = GetViewNoType();
+ Hash(View.GetData(), View.GetSize());
+ }
+
+ /** Copy the field into a buffer of exactly GetSize() bytes, including the type and name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the field into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ /**
+ * Whether this field is identical to the other field.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be performed with ValidateCompactBinary, except for field order and field name case.
+ */
+ ZENCORE_API bool Equals(const CbFieldView& Other) const;
+
+ /** Returns a view of the field, including the type and name when present. */
+ ZENCORE_API MemoryView GetView() const;
+
+ /**
+ * Try to get a view of the field as it would be serialized, such as by CopyTo.
+ *
+ * A serialized view is not available if the field has an externally-provided type.
+ * Access the serialized form of such fields using CopyTo or FCbFieldRef::Clone.
+ */
+ inline bool TryGetSerializedView(MemoryView& OutView) const
+ {
+ if (CbFieldTypeOps::HasFieldType(Type))
+ {
+ OutView = GetView();
+ return true;
+ }
+ return false;
+ }
+
+protected:
+ /** Returns a view of the name and value payload, which excludes the type. */
+ ZENCORE_API MemoryView GetViewNoType() const;
+
+ /** Returns a view of the value payload, which excludes the type and name. */
+ inline MemoryView GetPayloadView() const { return MemoryView(Payload, GetPayloadSize()); }
+
+ /** Returns the type of the field including flags. */
+ constexpr inline CbFieldType GetType() const { return Type; }
+
+ /** Returns the start of the value payload. */
+ constexpr inline const void* GetPayload() const { return Payload; }
+
+ /** Returns the end of the value payload. */
+ inline const void* GetPayloadEnd() const { return static_cast<const uint8_t*>(Payload) + GetPayloadSize(); }
+
+ /** Returns the size of the value payload in bytes, which is the field excluding the type and name. */
+ ZENCORE_API uint64_t GetPayloadSize() const;
+
+ /** Assign a field from a pointer to its data and an optional externally-provided type. */
+ inline void Assign(const void* InData, const CbFieldType InType)
+ {
+ static_assert(std::is_trivially_destructible<CbFieldView>::value,
+ "This optimization requires CbField to be trivially destructible!");
+ new (this) CbFieldView(InData, InType);
+ }
+
+private:
+ /** Parameters for converting to an integer. */
+ struct IntegerParams
+ {
+ /** Whether the output type has a sign bit. */
+ uint32_t IsSigned : 1;
+ /** Bits of magnitude. (7 for int8) */
+ uint32_t MagnitudeBits : 31;
+ };
+
+ /** Make integer params for the given integer type. */
+ template<typename IntType>
+ static constexpr inline IntegerParams MakeIntegerParams()
+ {
+ IntegerParams Params;
+ Params.IsSigned = IntType(-1) < IntType(0);
+ Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned;
+ return Params;
+ }
+
+ /**
+ * Access the field as the given integer type.
+ *
+ * Returns the provided default if the value cannot be represented in the output type.
+ */
+ template<typename IntType>
+ inline IntType AsInteger(IntType Default)
+ {
+ return IntType(AsInteger(uint64_t(Default), MakeIntegerParams<IntType>()));
+ }
+
+ ZENCORE_API uint64_t AsInteger(uint64_t Default, IntegerParams Params);
+
+ /** The field type, with the transient HasFieldType flag if the field contains its type. */
+ CbFieldType Type = CbFieldType::None;
+ /** The error (if any) that occurred on the last field access. */
+ CbFieldError Error = CbFieldError::None;
+ /** The number of bytes for the name stored before the payload. */
+ uint32_t NameLen = 0;
+ /** The value payload, which also points to the end of the name. */
+ const void* Payload = nullptr;
+};
+
+template<typename FieldType>
+class TCbFieldIterator : public FieldType
+{
+public:
+ /** Construct an empty field range. */
+ constexpr TCbFieldIterator() = default;
+
+ inline TCbFieldIterator& operator++()
+ {
+ const void* const PayloadEnd = FieldType::GetPayloadEnd();
+ const int64_t AtEndMask = int64_t(PayloadEnd == FieldsEnd) - 1;
+ const CbFieldType NextType = CbFieldType(int64_t(FieldType::GetType()) & AtEndMask);
+ const void* const NextField = reinterpret_cast<const void*>(int64_t(PayloadEnd) & AtEndMask);
+ const void* const NextFieldsEnd = reinterpret_cast<const void*>(int64_t(FieldsEnd) & AtEndMask);
+
+ FieldType::Assign(NextField, NextType);
+ FieldsEnd = NextFieldsEnd;
+ return *this;
+ }
+
+ inline TCbFieldIterator operator++(int)
+ {
+ TCbFieldIterator It(*this);
+ ++*this;
+ return It;
+ }
+
+ constexpr inline FieldType& operator*() { return *this; }
+ constexpr inline FieldType* operator->() { return this; }
+
+ /** Reset this to an empty field range. */
+ inline void Reset() { *this = TCbFieldIterator(); }
+
+ /** Returns the size of the fields in the range in bytes. */
+ ZENCORE_API uint64_t GetRangeSize() const;
+
+ /** Calculate the hash of every field in the range. */
+ ZENCORE_API IoHash GetRangeHash() const;
+ ZENCORE_API void GetRangeHash(IoHashStream& Hash) const;
+
+ using FieldType::Equals;
+
+ template<typename OtherFieldType>
+ constexpr inline bool Equals(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return FieldType::GetPayload() == Other.OtherFieldType::GetPayload() && FieldsEnd == Other.FieldsEnd;
+ }
+
+ template<typename OtherFieldType>
+ constexpr inline bool operator==(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return Equals(Other);
+ }
+
+ template<typename OtherFieldType>
+ constexpr inline bool operator!=(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return !Equals(Other);
+ }
+
+ /** Copy the field range into a buffer of exactly GetRangeSize() bytes. */
+ ZENCORE_API void CopyRangeTo(MutableMemoryView Buffer) const;
+
+ /** Invoke the visitor for every attachment in the field range. */
+ ZENCORE_API void IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const;
+
+ /** Create a view of every field in the range. */
+ inline MemoryView GetRangeView() const { return MemoryView(FieldType::GetView().GetData(), FieldsEnd); }
+
+ /**
+ * Try to get a view of every field in the range as they would be serialized.
+ *
+ * A serialized view is not available if the underlying fields have an externally-provided type.
+ * Access the serialized form of such ranges using CbFieldRefIterator::CloneRange.
+ */
+ inline bool TryGetSerializedRangeView(MemoryView& OutView) const
+ {
+ if (CbFieldTypeOps::HasFieldType(FieldType::GetType()))
+ {
+ OutView = GetRangeView();
+ return true;
+ }
+ return false;
+ }
+
+protected:
+ /** Construct a field range that contains exactly one field. */
+ constexpr inline explicit TCbFieldIterator(FieldType InField) : FieldType(std::move(InField)), FieldsEnd(FieldType::GetPayloadEnd()) {}
+
+ /**
+ * Construct a field range from the first field and a pointer to the end of the last field.
+ *
+ * @param InField The first field, or the default field if there are no fields.
+ * @param InFieldsEnd A pointer to the end of the payload of the last field, or null.
+ */
+ constexpr inline TCbFieldIterator(FieldType&& InField, const void* InFieldsEnd) : FieldType(std::move(InField)), FieldsEnd(InFieldsEnd)
+ {
+ }
+
+ /** Returns the end of the last field, or null for an iterator at the end. */
+ template<typename OtherFieldType>
+ static inline const void* GetFieldsEnd(const TCbFieldIterator<OtherFieldType>& It)
+ {
+ return It.FieldsEnd;
+ }
+
+private:
+ friend inline TCbFieldIterator begin(const TCbFieldIterator& Iterator) { return Iterator; }
+ friend inline TCbFieldIterator end(const TCbFieldIterator&) { return TCbFieldIterator(); }
+
+private:
+ template<typename OtherType>
+ friend class TCbFieldIterator;
+
+ friend class CbFieldViewIterator;
+
+ /** Pointer to the first byte past the end of the last field. Set to null at the end. */
+ const void* FieldsEnd = nullptr;
+};
+
+/**
+ * Iterator for CbField.
+ *
+ * @see CbFieldIterator
+ */
+class CbFieldViewIterator : public TCbFieldIterator<CbFieldView>
+{
+public:
+ constexpr CbFieldViewIterator() = default;
+
+ /** Construct a field range that contains exactly one field. */
+ static inline CbFieldViewIterator MakeSingle(const CbFieldView& Field) { return CbFieldViewIterator(Field); }
+
+ /**
+ * Construct a field range from a buffer containing zero or more valid fields.
+ *
+ * @param View A buffer containing zero or more valid fields.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ */
+ static inline CbFieldViewIterator MakeRange(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ return !View.IsEmpty() ? TCbFieldIterator(CbFieldView(View.GetData(), Type), View.GetDataEnd()) : CbFieldViewIterator();
+ }
+
+ /** Construct an iterator from another iterator. */
+ template<typename OtherFieldType>
+ inline CbFieldViewIterator(const TCbFieldIterator<OtherFieldType>& It)
+ : TCbFieldIterator(ImplicitConv<CbFieldView>(It), GetFieldsEnd(It))
+ {
+ }
+
+private:
+ using TCbFieldIterator::TCbFieldIterator;
+};
+
+/**
+ * Array of CbField that have no names.
+ *
+ * Accessing a field of the array requires iteration. Access by index is not provided because the
+ * cost of accessing an item by index scales linearly with the index.
+ *
+ * This type only provides a view into memory and does not perform any memory management itself.
+ * Use CbArrayRef to hold a reference to the underlying memory when necessary.
+ */
+class CbArrayView : protected CbFieldView
+{
+ friend class CbFieldView;
+
+public:
+ /** @see CbField::CbField */
+ using CbFieldView::CbFieldView;
+
+ /** Construct an array with no fields. */
+ ZENCORE_API CbArrayView();
+
+ /** Returns the number of items in the array. */
+ ZENCORE_API uint64_t Num() const;
+
+ /** Create an iterator for the fields of this array. */
+ ZENCORE_API CbFieldViewIterator CreateViewIterator() const;
+
+ /** Visit the fields of this array. */
+ ZENCORE_API void VisitFields(ICbVisitor& Visitor);
+
+ /** Access the array as an array field. */
+ inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); }
+
+ /** Construct an array from an array field. No type check is performed! */
+ static inline CbArrayView FromFieldView(const CbFieldView& Field) { return CbArrayView(Field); }
+
+ /** Returns the size of the array in bytes if serialized by itself with no name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the array if serialized by itself with no name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& Stream) const;
+
+ /**
+ * Whether this array is identical to the other array.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be done with the All mode to check these assumptions about the format of the inputs.
+ */
+ ZENCORE_API bool Equals(const CbArrayView& Other) const;
+
+ /** Copy the array into a buffer of exactly GetSize() bytes, with no name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the array into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ ///** Invoke the visitor for every attachment in the array. */
+ inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const
+ {
+ CreateViewIterator().IterateRangeAttachments(Visitor);
+ }
+
+ /** Returns a view of the array, including the type and name when present. */
+ using CbFieldView::GetView;
+
+private:
+ friend inline CbFieldViewIterator begin(const CbArrayView& Array) { return Array.CreateViewIterator(); }
+ friend inline CbFieldViewIterator end(const CbArrayView&) { return CbFieldViewIterator(); }
+
+ /** Construct an array from an array field. No type check is performed! Use via FromField. */
+ inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {}
+};
+
+class CbObjectView : protected CbFieldView
+{
+ friend class CbFieldView;
+
+public:
+ /** @see CbField::CbField */
+ using CbFieldView::CbFieldView;
+
+ /** Construct an object with no fields. */
+ ZENCORE_API CbObjectView();
+
+ /** Create an iterator for the fields of this object. */
+ ZENCORE_API CbFieldViewIterator CreateViewIterator() const;
+
+ /** Visit the fields of this object. */
+ ZENCORE_API void VisitFields(ICbVisitor& Visitor);
+
+ /**
+ * Find a field by case-sensitive name comparison.
+ *
+ * The cost of this operation scales linearly with the number of fields in the object. Prefer
+ * to iterate over the fields only once when consuming an object.
+ *
+ * @param Name The name of the field.
+ * @return The matching field if found, otherwise a field with no value.
+ */
+ ZENCORE_API CbFieldView FindView(std::string_view Name) const;
+
+ /** Find a field by case-insensitive name comparison. */
+ ZENCORE_API CbFieldView FindViewIgnoreCase(std::string_view Name) const;
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbFieldView operator[](std::string_view Name) const { return FindView(Name); }
+
+ /** Access the object as an object field. */
+ inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); }
+
+ /** Construct an object from an object field. No type check is performed! */
+ static inline CbObjectView FromFieldView(const CbFieldView& Field) { return CbObjectView(Field); }
+
+ /** Returns the size of the object in bytes if serialized by itself with no name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the object if serialized by itself with no name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& HashStream) const;
+
+ /**
+ * Whether this object is identical to the other object.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be done with the All mode to check these assumptions about the format of the inputs.
+ */
+ ZENCORE_API bool Equals(const CbObjectView& Other) const;
+
+ /** Copy the object into a buffer of exactly GetSize() bytes, with no name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the field into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ ///** Invoke the visitor for every attachment in the object. */
+ inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const
+ {
+ CreateViewIterator().IterateRangeAttachments(Visitor);
+ }
+
+ /** Returns a view of the object, including the type and name when present. */
+ using CbFieldView::GetView;
+
+ /** Whether the field has a value. */
+ using CbFieldView::operator bool;
+
+private:
+ friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); }
+ friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); }
+
+ /** Construct an object from an object field. No type check is performed! Use via FromField. */
+ inline explicit CbObjectView(const CbFieldView& Field) : CbFieldView(Field) {}
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+/** A reference to a function that is used to allocate buffers for compact binary data. */
+using BufferAllocator = std::function<UniqueBuffer(uint64_t Size)>;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** A wrapper that holds a reference to the buffer that contains its compact binary value. */
+template<typename BaseType>
+class CbBuffer : public BaseType
+{
+public:
+ /** Construct a default value. */
+ CbBuffer() = default;
+
+ /**
+ * Construct a value from a pointer to its data and an optional externally-provided type.
+ *
+ * @param ValueBuffer A buffer that exactly contains the value.
+ * @param Type HasFieldType means that ValueBuffer contains the type. Otherwise, use the given type.
+ */
+ inline explicit CbBuffer(SharedBuffer ValueBuffer, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ if (ValueBuffer)
+ {
+ BaseType::operator=(BaseType(ValueBuffer.GetData(), Type));
+ ZEN_ASSERT(ValueBuffer.GetView().Contains(BaseType::GetView()));
+ Buffer = std::move(ValueBuffer);
+ }
+ }
+
+ /** Construct a value that holds a reference to the buffer that contains it. */
+ inline CbBuffer(const BaseType& Value, SharedBuffer OuterBuffer) : BaseType(Value)
+ {
+ if (OuterBuffer)
+ {
+ ZEN_ASSERT(OuterBuffer.GetView().Contains(BaseType::GetView()));
+ Buffer = std::move(OuterBuffer);
+ }
+ }
+
+ /** Construct a value that holds a reference to the buffer of the outer that contains it. */
+ template<typename OtherBaseType>
+ inline CbBuffer(const BaseType& Value, CbBuffer<OtherBaseType> OuterRef) : CbBuffer(Value, std::move(OuterRef.Buffer))
+ {
+ }
+
+ /** Reset this to a default value and null buffer. */
+ inline void Reset() { *this = CbBuffer(); }
+
+ /** Whether this reference has ownership of the memory in its buffer. */
+ inline bool IsOwned() const { return Buffer && Buffer.IsOwned(); }
+
+ /** Clone the value, if necessary, to a buffer that this reference has ownership of. */
+ inline void MakeOwned()
+ {
+ if (!IsOwned())
+ {
+ UniqueBuffer MutableBuffer = UniqueBuffer::Alloc(BaseType::GetSize());
+ BaseType::CopyTo(MutableBuffer);
+ BaseType::operator=(BaseType(MutableBuffer.GetData()));
+ Buffer = std::move(MutableBuffer);
+ }
+ }
+
+ /** Returns a buffer that exactly contains this value. */
+ inline SharedBuffer GetBuffer() const
+ {
+ const MemoryView View = BaseType::GetView();
+ const SharedBuffer& OuterBuffer = GetOuterBuffer();
+ return View == OuterBuffer.GetView() ? OuterBuffer : SharedBuffer::MakeView(View, OuterBuffer);
+ }
+
+ /** Returns the outer buffer (if any) that contains this value. */
+ inline const SharedBuffer& GetOuterBuffer() const& { return Buffer; }
+ inline SharedBuffer GetOuterBuffer() && { return std::move(Buffer); }
+
+private:
+ template<typename OtherType>
+ friend class CbBuffer;
+
+ SharedBuffer Buffer;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Factory functions for types derived from CbBuffer.
+ *
+ * This uses the curiously recurring template pattern to construct the correct type of reference.
+ * The derived type inherits from CbBufferRef and this type to expose the factory functions.
+ */
+template<typename RefType, typename BaseType>
+class CbBufferFactory
+{
+public:
+ /** Construct a value from an owned clone of its memory. */
+ static inline RefType Clone(const void* const Data) { return Clone(BaseType(Data)); }
+
+ /** Construct a value from an owned clone of its memory. */
+ static inline RefType Clone(const BaseType& Value)
+ {
+ RefType Ref = MakeView(Value);
+ Ref.MakeOwned();
+ return Ref;
+ }
+
+ /** Construct a value from a read-only view of its memory and its optional outer buffer. */
+ static inline RefType MakeView(const void* const Data, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return MakeView(BaseType(Data), std::move(OuterBuffer));
+ }
+
+ /** Construct a value from a read-only view of its memory and its optional outer buffer. */
+ static inline RefType MakeView(const BaseType& Value, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return RefType(Value, std::move(OuterBuffer));
+ }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class CbArray;
+class CbObject;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A field that can hold a reference to the memory that contains it.
+ *
+ * @see CbBufferRef
+ */
+class CbField : public CbBuffer<CbFieldView>, public CbBufferFactory<CbField, CbFieldView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Access the field as an object. Defaults to an empty object on error. */
+ inline CbObject AsObject() &;
+
+ /** Access the field as an object. Defaults to an empty object on error. */
+ inline CbObject AsObject() &&;
+
+ /** Access the field as an array. Defaults to an empty array on error. */
+ inline CbArray AsArray() &;
+
+ /** Access the field as an array. Defaults to an empty array on error. */
+ inline CbArray AsArray() &&;
+};
+
+/**
+ * Iterator for CbFieldRef.
+ *
+ * @see CbFieldIterator
+ */
+class CbFieldIterator : public TCbFieldIterator<CbField>
+{
+public:
+ /** Construct a field range from an owned clone of a range. */
+ ZENCORE_API static CbFieldIterator CloneRange(const CbFieldViewIterator& It);
+
+ /** Construct a field range from an owned clone of a range. */
+ static inline CbFieldIterator CloneRange(const CbFieldIterator& It) { return CloneRange(CbFieldViewIterator(It)); }
+
+ /** Construct a field range that contains exactly one field. */
+ static inline CbFieldIterator MakeSingle(CbField Field) { return CbFieldIterator(std::move(Field)); }
+
+ /**
+ * Construct a field range from a buffer containing zero or more valid fields.
+ *
+ * @param Buffer A buffer containing zero or more valid fields.
+ * @param Type HasFieldType means that Buffer contains the type. Otherwise, use the given type.
+ */
+ static inline CbFieldIterator MakeRange(SharedBuffer Buffer, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ if (Buffer.GetSize())
+ {
+ const void* const DataEnd = Buffer.GetView().GetDataEnd();
+ return CbFieldIterator(CbField(std::move(Buffer), Type), DataEnd);
+ }
+ return CbFieldIterator();
+ }
+
+ /** Construct a field range from an iterator and its optional outer buffer. */
+ static inline CbFieldIterator MakeRangeView(const CbFieldViewIterator& It, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return CbFieldIterator(CbField(It, std::move(OuterBuffer)), GetFieldsEnd(It));
+ }
+
+ /** Construct an empty field range. */
+ constexpr CbFieldIterator() = default;
+
+ /** Clone the range, if necessary, to a buffer that this reference has ownership of. */
+ inline void MakeRangeOwned()
+ {
+ if (!IsOwned())
+ {
+ *this = CloneRange(*this);
+ }
+ }
+
+ /** Returns a buffer that exactly contains the field range. */
+ ZENCORE_API SharedBuffer GetRangeBuffer() const;
+
+private:
+ using TCbFieldIterator::TCbFieldIterator;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An array that can hold a reference to the memory that contains it.
+ *
+ * @see CbBuffer
+ */
+class CbArray : public CbBuffer<CbArrayView>, public CbBufferFactory<CbArray, CbArrayView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Create an iterator for the fields of this array. */
+ inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); }
+
+ /** Access the array as an array field. */
+ inline CbField AsField() const& { return CbField(CbArrayView::AsFieldView(), *this); }
+
+ /** Access the array as an array field. */
+ inline CbField AsField() && { return CbField(CbArrayView::AsFieldView(), std::move(*this)); }
+
+private:
+ friend inline CbFieldIterator begin(const CbArray& Array) { return Array.CreateIterator(); }
+ friend inline CbFieldIterator end(const CbArray&) { return CbFieldIterator(); }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An object that can hold a reference to the memory that contains it.
+ *
+ * @see CbBuffer
+ */
+class CbObject : public CbBuffer<CbObjectView>, public CbBufferFactory<CbObject, CbObjectView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Create an iterator for the fields of this object. */
+ inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); }
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbField Find(std::string_view Name) const
+ {
+ if (CbFieldView Field = FindView(Name))
+ {
+ return CbField(Field, *this);
+ }
+ return CbField();
+ }
+
+ /** Find a field by case-insensitive name comparison. */
+ inline CbField FindIgnoreCase(std::string_view Name) const
+ {
+ if (CbFieldView Field = FindIgnoreCase(Name))
+ {
+ return CbField(Field, *this);
+ }
+ return CbField();
+ }
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbFieldView operator[](std::string_view Name) const { return Find(Name); }
+
+ /** Access the object as an object field. */
+ inline CbField AsField() const& { return CbField(CbObjectView::AsFieldView(), *this); }
+
+ /** Access the object as an object field. */
+ inline CbField AsField() && { return CbField(CbObjectView::AsFieldView(), std::move(*this)); }
+
+private:
+ friend inline CbFieldIterator begin(const CbObject& Object) { return Object.CreateIterator(); }
+ friend inline CbFieldIterator end(const CbObject&) { return CbFieldIterator(); }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+inline CbObject
+CbField::AsObject() &
+{
+ return IsObject() ? CbObject(AsObjectView(), *this) : CbObject();
+}
+
+inline CbObject
+CbField::AsObject() &&
+{
+ return IsObject() ? CbObject(AsObjectView(), std::move(*this)) : CbObject();
+}
+
+inline CbArray
+CbField::AsArray() &
+{
+ return IsArray() ? CbArray(AsArrayView(), *this) : CbArray();
+}
+
+inline CbArray
+CbField::AsArray() &&
+{
+ return IsArray() ? CbArray(AsArrayView(), std::move(*this)) : CbArray();
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator);
+
+inline CbObject
+LoadCompactBinaryObject(IoBuffer Payload)
+{
+ return CbObject{SharedBuffer::MakeView(Payload.Data(), Payload.Size())};
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Determine the size in bytes of the compact binary field at the start of the view.
+ *
+ * This may be called on an incomplete or invalid field, in which case the returned size is zero.
+ * A size can always be extracted from a valid field with no name if a view of at least the first
+ * 10 bytes is provided, regardless of field size. For fields with names, the size of view needed
+ * to calculate a size is at most 10 + MaxNameLen + MeasureVarUInt(MaxNameLen).
+ *
+ * This function can be used when streaming a field, for example, to determine the size of buffer
+ * to fill before attempting to construct a field from it.
+ *
+ * @param View A memory view that may contain the start of a field.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ */
+ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType);
+
+/**
+ * Try to determine the type and size of the compact binary field at the start of the view.
+ *
+ * This may be called on an incomplete or invalid field, in which case it will return false, with
+ * OutSize being 0 for invalid fields, otherwise the minimum view size necessary to make progress
+ * in measuring the field on the next call to this function.
+ *
+ * @note A return of true from this function does not indicate that the entire field is valid.
+ *
+ * @param InView A memory view that may contain the start of a field.
+ * @param OutType The type (with flags) of the field. None is written until a value is available.
+ * @param OutSize The total field size for a return of true, 0 for invalid fields, or the size to
+ * make progress in measuring the field on the next call to this function.
+ * @param InType HasFieldType means that InView contains the type. Otherwise, use the given type.
+ * @return true if the size of the field was determined, otherwise false.
+ */
+ZENCORE_API bool TryMeasureCompactBinary(MemoryView InView,
+ CbFieldType& OutType,
+ uint64_t& OutSize,
+ CbFieldType InType = CbFieldType::HasFieldType);
+
+inline CbFieldViewIterator
+begin(CbFieldView& View)
+{
+ if (View.IsArray())
+ {
+ return View.AsArrayView().CreateViewIterator();
+ }
+ else if (View.IsObject())
+ {
+ return View.AsObjectView().CreateViewIterator();
+ }
+
+ return CbFieldViewIterator();
+}
+
+inline CbFieldViewIterator
+end(CbFieldView&)
+{
+ return CbFieldViewIterator();
+}
+
+void uson_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h
new file mode 100644
index 000000000..83d4309f7
--- /dev/null
+++ b/zencore/include/zencore/compactbinarybuilder.h
@@ -0,0 +1,633 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+
+#include <zencore/enumflags.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/refcount.h>
+#include <zencore/sha1.h>
+
+#include <atomic>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class CbAttachment;
+class BinaryWriter;
+
+/**
+ * A writer for compact binary object, arrays, and fields.
+ *
+ * The writer produces a sequence of fields that can be saved to a provided memory buffer or into
+ * a new owned buffer. The typical use case is to write a single object, which can be accessed by
+ * calling Save().AsObjectRef() or Save(Buffer).AsObject().
+ *
+ * The writer will assert on most incorrect usage and will always produce valid compact binary if
+ * provided with valid input. The writer does not check for invalid UTF-8 string encoding, object
+ * fields with duplicate names, or invalid compact binary being copied from another source.
+ *
+ * It is most convenient to use the streaming API for the writer, as demonstrated in the example.
+ *
+ * When writing a small amount of compact binary data, TCbWriter can be more efficient as it uses
+ * a fixed-size stack buffer for storage before spilling onto the heap.
+ *
+ * @see TCbWriter
+ *
+ * Example:
+ *
+ * CbObjectRef WriteObject()
+ * {
+ * CbWriter<256> Writer;
+ * Writer.BeginObject();
+ *
+ * Writer << "Resize" << true;
+ * Writer << "MaxWidth" << 1024;
+ * Writer << "MaxHeight" << 1024;
+ *
+ * Writer.BeginArray();
+ * Writer << "FormatA" << "FormatB" << "FormatC";
+ * Writer.EndArray();
+ *
+ * Writer.EndObject();
+ * return Writer.Save().AsObjectRef();
+ * }
+ */
+class CbWriter
+{
+public:
+ ZENCORE_API CbWriter();
+ ZENCORE_API ~CbWriter();
+
+ CbWriter(const CbWriter&) = delete;
+ CbWriter& operator=(const CbWriter&) = delete;
+
+ /** Empty the writer without releasing any allocated memory. */
+ ZENCORE_API void Reset();
+
+ /**
+ * Serialize the field(s) to an owned buffer and return it as an iterator.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ * The writer remains valid for further use when this function returns.
+ */
+ ZENCORE_API CbFieldIterator Save();
+
+ /**
+ * Serialize the field(s) to memory.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ * The writer remains valid for further use when this function returns.
+ *
+ * @param Buffer A mutable memory view to write to. Must be exactly GetSaveSize() bytes.
+ * @return An iterator for the field(s) written to the buffer.
+ */
+ ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer);
+
+ ZENCORE_API void Save(BinaryWriter& Writer);
+
+ /**
+ * The size of buffer (in bytes) required to serialize the fields that have been written.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ */
+ ZENCORE_API uint64_t GetSaveSize() const;
+
+ /**
+ * Sets the name of the next field to be written.
+ *
+ * It is not valid to call this function when writing a field inside an array.
+ * Names must be valid UTF-8 and must be unique within an object.
+ */
+ ZENCORE_API CbWriter& SetName(std::string_view Name);
+
+ /** Copy the value (not the name) of an existing field. */
+ inline void AddField(std::string_view Name, const CbFieldView& Value)
+ {
+ SetName(Name);
+ AddField(Value);
+ }
+
+ ZENCORE_API void AddField(const CbFieldView& Value);
+
+ /** Copy the value (not the name) of an existing field. Holds a reference if owned. */
+ inline void AddField(std::string_view Name, const CbField& Value)
+ {
+ SetName(Name);
+ AddField(Value);
+ }
+ ZENCORE_API void AddField(const CbField& Value);
+
+ /** Begin a new object. Must have a matching call to EndObject. */
+ inline void BeginObject(std::string_view Name)
+ {
+ SetName(Name);
+ BeginObject();
+ }
+ ZENCORE_API void BeginObject();
+ /** End an object after its fields have been written. */
+ ZENCORE_API void EndObject();
+
+ /** Copy the value (not the name) of an existing object. */
+ inline void AddObject(std::string_view Name, const CbObjectView& Value)
+ {
+ SetName(Name);
+ AddObject(Value);
+ }
+ ZENCORE_API void AddObject(const CbObjectView& Value);
+ /** Copy the value (not the name) of an existing object. Holds a reference if owned. */
+ inline void AddObject(std::string_view Name, const CbObject& Value)
+ {
+ SetName(Name);
+ AddObject(Value);
+ }
+ ZENCORE_API void AddObject(const CbObject& Value);
+
+ /** Begin a new array. Must have a matching call to EndArray. */
+ inline void BeginArray(std::string_view Name)
+ {
+ SetName(Name);
+ BeginArray();
+ }
+ ZENCORE_API void BeginArray();
+ /** End an array after its fields have been written. */
+ ZENCORE_API void EndArray();
+
+ /** Copy the value (not the name) of an existing array. */
+ inline void AddArray(std::string_view Name, const CbArrayView& Value)
+ {
+ SetName(Name);
+ AddArray(Value);
+ }
+ ZENCORE_API void AddArray(const CbArrayView& Value);
+ /** Copy the value (not the name) of an existing array. Holds a reference if owned. */
+ inline void AddArray(std::string_view Name, const CbArray& Value)
+ {
+ SetName(Name);
+ AddArray(Value);
+ }
+ ZENCORE_API void AddArray(const CbArray& Value);
+
+ /** Write a null field. */
+ inline void AddNull(std::string_view Name)
+ {
+ SetName(Name);
+ AddNull();
+ }
+ ZENCORE_API void AddNull();
+
+ /** Write a binary field by copying Size bytes from Value. */
+ inline void AddBinary(std::string_view Name, const void* Value, uint64_t Size)
+ {
+ SetName(Name);
+ AddBinary(Value, Size);
+ }
+ ZENCORE_API void AddBinary(const void* Value, uint64_t Size);
+ /** Write a binary field by copying the view. */
+ inline void AddBinary(std::string_view Name, MemoryView Value)
+ {
+ SetName(Name);
+ AddBinary(Value);
+ }
+ inline void AddBinary(MemoryView Value) { AddBinary(Value.GetData(), Value.GetSize()); }
+
+ /** Write a binary field by copying the buffer. Holds a reference if owned. */
+ inline void AddBinary(std::string_view Name, IoBuffer Value)
+ {
+ SetName(Name);
+ AddBinary(std::move(Value));
+ }
+ ZENCORE_API void AddBinary(IoBuffer Value);
+ ZENCORE_API void AddBinary(SharedBuffer Value);
+
+ /** Write a string field by copying the UTF-8 value. */
+ inline void AddString(std::string_view Name, std::string_view Value)
+ {
+ SetName(Name);
+ AddString(Value);
+ }
+ ZENCORE_API void AddString(std::string_view Value);
+ /** Write a string field by converting the UTF-16 value to UTF-8. */
+ inline void AddString(std::string_view Name, std::wstring_view Value)
+ {
+ SetName(Name);
+ AddString(Value);
+ }
+ ZENCORE_API void AddString(std::wstring_view Value);
+
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, int32_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(int32_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, int64_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(int64_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, uint32_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(uint32_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, uint64_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(uint64_t Value);
+
+ /** Write a float field from a 32-bit float value. */
+ inline void AddFloat(std::string_view Name, float Value)
+ {
+ SetName(Name);
+ AddFloat(Value);
+ }
+ ZENCORE_API void AddFloat(float Value);
+
+ /** Write a float field from a 64-bit float value. */
+ inline void AddFloat(std::string_view Name, double Value)
+ {
+ SetName(Name);
+ AddFloat(Value);
+ }
+ ZENCORE_API void AddFloat(double Value);
+
+ /** Write a bool field. */
+ inline void AddBool(std::string_view Name, bool bValue)
+ {
+ SetName(Name);
+ AddBool(bValue);
+ }
+ ZENCORE_API void AddBool(bool bValue);
+
+ /** Write a field referencing a compact binary attachment by its hash. */
+ inline void AddCompactBinaryAttachment(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddCompactBinaryAttachment(Value);
+ }
+ ZENCORE_API void AddCompactBinaryAttachment(const IoHash& Value);
+
+ /** Write a field referencing a binary attachment by its hash. */
+ inline void AddBinaryAttachment(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddBinaryAttachment(Value);
+ }
+ ZENCORE_API void AddBinaryAttachment(const IoHash& Value);
+
+ /** Write a field referencing the attachment by its hash. */
+ inline void AddAttachment(std::string_view Name, const CbAttachment& Attachment)
+ {
+ SetName(Name);
+ AddAttachment(Attachment);
+ }
+ ZENCORE_API void AddAttachment(const CbAttachment& Attachment);
+
+ /** Write a hash field. */
+ inline void AddHash(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddHash(Value);
+ }
+ ZENCORE_API void AddHash(const IoHash& Value);
+
+ /** Write a UUID field. */
+ inline void AddUuid(std::string_view Name, const Guid& Value)
+ {
+ SetName(Name);
+ AddUuid(Value);
+ }
+ ZENCORE_API void AddUuid(const Guid& Value);
+
+ /** Write an ObjectId field. */
+ inline void AddObjectId(std::string_view Name, const Oid& Value)
+ {
+ SetName(Name);
+ AddObjectId(Value);
+ }
+ ZENCORE_API void AddObjectId(const Oid& Value);
+
+ /** Write a date/time field with the specified count of 100ns ticks since the epoch. */
+ inline void AddDateTimeTicks(std::string_view Name, int64_t Ticks)
+ {
+ SetName(Name);
+ AddDateTimeTicks(Ticks);
+ }
+ ZENCORE_API void AddDateTimeTicks(int64_t Ticks);
+
+ /** Write a date/time field. */
+ inline void AddDateTime(std::string_view Name, DateTime Value)
+ {
+ SetName(Name);
+ AddDateTime(Value);
+ }
+ ZENCORE_API void AddDateTime(DateTime Value);
+
+ /** Write a time span field with the specified count of 100ns ticks. */
+ inline void AddTimeSpanTicks(std::string_view Name, int64_t Ticks)
+ {
+ SetName(Name);
+ AddTimeSpanTicks(Ticks);
+ }
+ ZENCORE_API void AddTimeSpanTicks(int64_t Ticks);
+
+ /** Write a time span field. */
+ inline void AddTimeSpan(std::string_view Name, TimeSpan Value)
+ {
+ SetName(Name);
+ AddTimeSpan(Value);
+ }
+ ZENCORE_API void AddTimeSpan(TimeSpan Value);
+
+ /** Private flags that are public to work with ENUM_CLASS_FLAGS. */
+ enum class StateFlags : uint8_t;
+
+protected:
+ /** Reserve the specified size up front until the format is optimized. */
+ ZENCORE_API explicit CbWriter(int64_t InitialSize);
+
+private:
+ friend CbWriter& operator<<(CbWriter& Writer, std::string_view NameOrValue);
+
+ /** Begin writing a field. May be called twice for named fields. */
+ void BeginField();
+
+ /** Finish writing a field by writing its type. */
+ void EndField(CbFieldType Type);
+
+ /** Set the field name if valid in this state, otherwise write add a string field. */
+ ZENCORE_API void SetNameOrAddString(std::string_view NameOrValue);
+
+ /** Returns a view of the name of the active field, if any, otherwise the empty view. */
+ std::string_view GetActiveName() const;
+
+ /** Remove field types after the first to make the sequence uniform. */
+ void MakeFieldsUniform(int64_t FieldBeginOffset, int64_t FieldEndOffset);
+
+ /** State of the object, array, or top-level field being written. */
+ struct WriterState
+ {
+ StateFlags Flags{};
+ /** The type of the fields in the sequence if uniform, otherwise None. */
+ CbFieldType UniformType{};
+ /** The offset of the start of the current field. */
+ int64_t Offset{};
+ /** The number of fields written in this state. */
+ uint64_t Count{};
+ };
+
+private:
+ // This is a prototype-quality format for the writer. Using an array of bytes is inefficient,
+ // and will lead to many unnecessary copies and moves of the data to resize the array, insert
+ // object and array sizes, and remove field types for uniform objects and uniform arrays. The
+ // optimized format will be a list of power-of-two blocks and an optional first block that is
+ // provided externally, such as on the stack. That format will store the offsets that require
+ // object or array sizes to be inserted and field types to be removed, and will perform those
+ // operations only when saving to a buffer.
+ std::vector<uint8_t> Data;
+ std::vector<WriterState> States;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A writer for compact binary object, arrays, and fields that uses a fixed-size stack buffer.
+ *
+ * @see CbWriter
+ */
+template<uint32_t InlineBufferSize>
+class FixedCbWriter : public CbWriter
+{
+public:
+ inline FixedCbWriter() : CbWriter(InlineBufferSize) {}
+
+ FixedCbWriter(const FixedCbWriter&) = delete;
+ FixedCbWriter& operator=(const FixedCbWriter&) = delete;
+
+private:
+ // Reserve the inline buffer now even though we are unable to use it. This will avoid causing
+ // new stack overflows when this functionality is properly implemented in the future.
+ uint8_t Buffer[InlineBufferSize];
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class CbObjectWriter : public CbWriter
+{
+public:
+ CbObjectWriter() { BeginObject(); }
+
+ ZENCORE_API CbObject Save()
+ {
+ Finalize();
+ return CbWriter::Save().AsObject();
+ }
+
+ ZENCORE_API void Save(BinaryWriter& Writer)
+ {
+ Finalize();
+ return CbWriter::Save(Writer);
+ }
+
+ uint64_t GetSaveSize() = delete;
+
+ void Finalize()
+ {
+ if (m_Finalized == false)
+ {
+ EndObject();
+ m_Finalized = true;
+ }
+ }
+
+ CbObjectWriter(const CbWriter&) = delete;
+ CbObjectWriter& operator=(const CbWriter&) = delete;
+
+private:
+ bool m_Finalized = false;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Write the field name if valid in this state, otherwise write the string value. */
+inline CbWriter&
+operator<<(CbWriter& Writer, std::string_view NameOrValue)
+{
+ Writer.SetNameOrAddString(NameOrValue);
+ return Writer;
+}
+
+/** Write the field name if valid in this state, otherwise write the string value. */
+inline CbWriter&
+operator<<(CbWriter& Writer, const char* NameOrValue)
+{
+ return Writer << std::string_view(NameOrValue);
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbFieldView& Value)
+{
+ Writer.AddField(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbField& Value)
+{
+ Writer.AddField(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbObjectView& Value)
+{
+ Writer.AddObject(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbObject& Value)
+{
+ Writer.AddObject(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbArrayView& Value)
+{
+ Writer.AddArray(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbArray& Value)
+{
+ Writer.AddArray(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, nullptr_t)
+{
+ Writer.AddNull();
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, std::wstring_view Value)
+{
+ Writer.AddString(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const wchar_t* Value)
+{
+ Writer.AddString(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, int32_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, int64_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, uint32_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, uint64_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, float Value)
+{
+ Writer.AddFloat(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, double Value)
+{
+ Writer.AddFloat(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, bool Value)
+{
+ Writer.AddBool(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbAttachment& Attachment)
+{
+ Writer.AddAttachment(Attachment);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const IoHash& Value)
+{
+ Writer.AddHash(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const Guid& Value)
+{
+ Writer.AddUuid(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const Oid& Value)
+{
+ Writer.AddObjectId(Value);
+ return Writer;
+}
+
+ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value);
+ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void usonbuilder_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinarypackage.h b/zencore/include/zencore/compactbinarypackage.h
new file mode 100644
index 000000000..c98ab047f
--- /dev/null
+++ b/zencore/include/zencore/compactbinarypackage.h
@@ -0,0 +1,305 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/iohash.h>
+
+#include <functional>
+#include <span>
+
+namespace zen {
+
+class CbWriter;
+class BinaryReader;
+class BinaryWriter;
+class IoBuffer;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An attachment is either binary or compact binary and is identified by its hash.
+ *
+ * A compact binary attachment is also a valid binary attachment and may be accessed as binary.
+ *
+ * Attachments are serialized as one or two compact binary fields with no name. A Binary field is
+ * written first with its content. The content hash is omitted when the content size is zero, and
+ * is otherwise written as a BinaryReference or CompactBinaryReference depending on the type.
+ */
+class CbAttachment
+{
+public:
+ /** Construct a null attachment. */
+ CbAttachment() = default;
+
+ /** Construct a compact binary attachment. Value is cloned if not owned. */
+ inline explicit CbAttachment(CbFieldIterator Value) : CbAttachment(std::move(Value), nullptr) {}
+
+ /** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */
+ inline explicit CbAttachment(CbFieldIterator Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {}
+
+ /** Construct a binary attachment. Value is cloned if not owned. */
+ inline explicit CbAttachment(SharedBuffer Value) : CbAttachment(std::move(Value), nullptr) {}
+
+ /** Construct a binary attachment. Value is cloned if not owned. Hash must match Value. */
+ inline explicit CbAttachment(SharedBuffer Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {}
+
+ /** Reset this to a null attachment. */
+ inline void Reset() { *this = CbAttachment(); }
+
+ /** Whether the attachment has a value. */
+ inline explicit operator bool() const { return !IsNull(); }
+
+ /** Whether the attachment has a value. */
+ inline bool IsNull() const { return !Buffer; }
+
+ /** Access the attachment as binary. Defaults to a null buffer on error. */
+ ZENCORE_API SharedBuffer AsBinaryView() const;
+
+ /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */
+ ZENCORE_API CbFieldIterator AsCompactBinary() const;
+
+ /** Returns whether the attachment is binary or compact binary. */
+ inline bool IsBinary() const { return !Buffer.IsNull(); }
+
+ /** Returns whether the attachment is compact binary. */
+ inline bool IsCompactBinary() const { return CompactBinary.HasValue(); }
+
+ /** Returns the hash of the attachment value. */
+ inline const IoHash& GetHash() const { return Hash; }
+
+ /** Compares attachments by their hash. Any discrepancy in type must be handled externally. */
+ inline bool operator==(const CbAttachment& Attachment) const { return Hash == Attachment.Hash; }
+ inline bool operator!=(const CbAttachment& Attachment) const { return Hash != Attachment.Hash; }
+ inline bool operator<(const CbAttachment& Attachment) const { return Hash < Attachment.Hash; }
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ *
+ * The attachment references the input iterator if it is owned, and otherwise clones the value.
+ *
+ * The iterator is advanced as attachment fields are consumed from it.
+ */
+ ZENCORE_API void Load(CbFieldIterator& Fields);
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ */
+ ZENCORE_API void Load(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ */
+ ZENCORE_API void Load(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ /** Save the attachment into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(CbWriter& Writer) const;
+
+ /** Save the attachment into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(BinaryWriter& Writer) const;
+
+private:
+ ZENCORE_API CbAttachment(CbFieldIterator Value, const IoHash* Hash);
+ ZENCORE_API CbAttachment(SharedBuffer Value, const IoHash* Hash);
+
+ /** An owned buffer containing the binary or compact binary data. */
+ SharedBuffer Buffer;
+ /** A field iterator that is valid only for compact binary attachments. */
+ CbFieldViewIterator CompactBinary;
+ /** A hash of the attachment value. */
+ IoHash Hash;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A package is a compact binary object with attachments for its external references.
+ *
+ * A package is basically a Merkle tree with compact binary as its root and other non-leaf nodes,
+ * and either binary or compact binary as its leaf nodes. A node references its child nodes using
+ * BinaryHash or FieldHash fields in its compact binary representation.
+ *
+ * It is invalid for a package to include attachments that are not referenced by its object or by
+ * one of its referenced compact binary attachments. When attachments are added explicitly, it is
+ * the responsibility of the package creator to follow this requirement. Attachments that are not
+ * referenced may not survive a round-trip through certain storage systems.
+ *
+ * It is valid for a package to exclude referenced attachments, but then it is the responsibility
+ * of the package consumer to have a mechanism for resolving those references when necessary.
+ *
+ * A package is serialized as a sequence of compact binary fields with no name. The object may be
+ * both preceded and followed by attachments. The object itself is written as an Object field and
+ * followed by its hash in a CompactBinaryReference field when the object is non-empty. A package
+ * ends with a Null field. The canonical order of components is the object and its hash, followed
+ * by the attachments ordered by hash, followed by a Null field. It is valid for the a package to
+ * have its components serialized in any order, provided there is at most one object and the null
+ * field is written last.
+ */
+class CbPackage
+{
+public:
+ /**
+ * A function that resolves a hash to a buffer containing the data matching that hash.
+ *
+ * The resolver may return a null buffer to skip resolving an attachment for the hash.
+ */
+ using AttachmentResolver = std::function<SharedBuffer(const IoHash& Hash)>;
+
+ /** Construct a null package. */
+ CbPackage() = default;
+
+ /**
+ * Construct a package from a root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ */
+ inline explicit CbPackage(CbObject InObject) { SetObject(std::move(InObject)); }
+
+ /**
+ * Construct a package from a root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline explicit CbPackage(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), InResolver); }
+
+ /**
+ * Construct a package from a root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ */
+ inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), InObjectHash); }
+
+ /**
+ * Construct a package from a root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver)
+ {
+ SetObject(std::move(InObject), InObjectHash, InResolver);
+ }
+
+ /** Reset this to a null package. */
+ inline void Reset() { *this = CbPackage(); }
+
+ /** Whether the package has a non-empty object or attachments. */
+ inline explicit operator bool() const { return !IsNull(); }
+
+ /** Whether the package has an empty object and no attachments. */
+ inline bool IsNull() const { return !Object.CreateIterator() && Attachments.size() == 0; }
+
+ /** Returns the compact binary object for the package. */
+ inline const CbObject& GetObject() const { return Object; }
+
+ /** Returns the has of the compact binary object for the package. */
+ inline const IoHash& GetObjectHash() const { return ObjectHash; }
+
+ /**
+ * Set the root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ */
+ inline void SetObject(CbObject InObject) { SetObject(std::move(InObject), nullptr, nullptr); }
+
+ /**
+ * Set the root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline void SetObject(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), nullptr, &InResolver); }
+
+ /**
+ * Set the root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ */
+ inline void SetObject(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), &InObjectHash, nullptr); }
+
+ /**
+ * Set the root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline void SetObject(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver)
+ {
+ SetObject(std::move(InObject), &InObjectHash, &InResolver);
+ }
+
+ /** Returns the attachments in this package. */
+ inline std::span<const CbAttachment> GetAttachments() const { return Attachments; }
+
+ /**
+ * Find an attachment by its hash.
+ *
+ * @return The attachment, or null if the attachment is not found.
+ * @note The returned pointer is only valid until the attachments on this package are modified.
+ */
+ ZENCORE_API const CbAttachment* FindAttachment(const IoHash& Hash) const;
+
+ /** Find an attachment if it exists in the package. */
+ inline const CbAttachment* FindAttachment(const CbAttachment& Attachment) const { return FindAttachment(Attachment.GetHash()); }
+
+ /** Add the attachment to this package. */
+ inline void AddAttachment(const CbAttachment& Attachment) { AddAttachment(Attachment, nullptr); }
+
+ /** Add the attachment to this package, along with any references that can be resolved. */
+ inline void AddAttachment(const CbAttachment& Attachment, AttachmentResolver Resolver) { AddAttachment(Attachment, &Resolver); }
+
+ /**
+ * Remove an attachment by hash.
+ *
+ * @return Number of attachments removed, which will be either 0 or 1.
+ */
+ ZENCORE_API int32_t RemoveAttachment(const IoHash& Hash);
+ inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); }
+
+ /** Compares packages by their object and attachment hashes. */
+ ZENCORE_API bool Equals(const CbPackage& Package) const;
+ inline bool operator==(const CbPackage& Package) const { return Equals(Package); }
+ inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); }
+
+ /**
+ * Load the object and attachments from compact binary as written by Save.
+ *
+ * The object and attachments reference the input iterator, if it is owned, and otherwise clones
+ * the object and attachments individually to make owned copies.
+ *
+ * The iterator is advanced as object and attachment fields are consumed from it.
+ */
+ ZENCORE_API void Load(CbFieldIterator& Fields);
+
+ ZENCORE_API void Load(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ ZENCORE_API void Load(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ /** Save the object and attachments into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(CbWriter& Writer) const;
+
+ /** Save the object and attachments into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(BinaryWriter& Writer) const;
+
+private:
+ ZENCORE_API void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver);
+ ZENCORE_API void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver);
+
+ void GatherAttachments(const CbFieldViewIterator& Fields, AttachmentResolver Resolver);
+
+ /** Attachments ordered by their hash. */
+ std::vector<CbAttachment> Attachments;
+ CbObject Object;
+ IoHash ObjectHash;
+};
+
+void usonpackage_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinaryvalidation.h b/zencore/include/zencore/compactbinaryvalidation.h
new file mode 100644
index 000000000..3a3f432be
--- /dev/null
+++ b/zencore/include/zencore/compactbinaryvalidation.h
@@ -0,0 +1,192 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/enumflags.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/refcount.h>
+#include <zencore/sha1.h>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+/** Flags for validating compact binary data. */
+enum class CbValidateMode : uint32_t
+{
+ /** Skip validation if no other validation modes are enabled. */
+ None = 0,
+
+ /**
+ * Validate that the value can be read and stays inside the bounds of the memory view.
+ *
+ * This is the minimum level of validation required to be able to safely read a field, array,
+ * or object without the risk of crashing or reading out of bounds.
+ */
+ Default = 1 << 0,
+
+ /**
+ * Validate that object fields have unique non-empty names and array fields have no names.
+ *
+ * Name validation failures typically do not inhibit reading the input, but duplicated fields
+ * cannot be looked up by name other than the first, and converting to other data formats can
+ * fail in the presence of naming issues.
+ */
+ Names = 1 << 1,
+
+ /**
+ * Validate that fields are serialized in the canonical format.
+ *
+ * Format validation failures typically do not inhibit reading the input. Values that fail in
+ * this mode require more memory than in the canonical format, and comparisons of such values
+ * for equality are not reliable. Examples of failures include uniform arrays or objects that
+ * were not encoded uniformly, variable-length integers that could be encoded in fewer bytes,
+ * or 64-bit floats that could be encoded in 32 bits without loss of precision.
+ */
+ Format = 1 << 2,
+
+ /**
+ * Validate that there is no padding after the value before the end of the memory view.
+ *
+ * Padding validation failures have no impact on the ability to read the input, but are using
+ * more memory than necessary.
+ */
+ Padding = 1 << 3,
+
+ /**
+ * Validate that a package or attachment has the expected fields and matches its saved hashes.
+ */
+ Package = 1 << 4,
+
+ /** Perform all validation described above. */
+ All = Default | Names | Format | Padding | Package,
+};
+
+ENUM_CLASS_FLAGS(CbValidateMode);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Flags for compact binary validation errors. Multiple flags may be combined. */
+enum class CbValidateError : uint32_t
+{
+ /** The input had no validation errors. */
+ None = 0,
+
+ // Mode: Default
+
+ /** The input cannot be read without reading out of bounds. */
+ OutOfBounds = 1 << 0,
+ /** The input has a field with an unrecognized or invalid type. */
+ InvalidType = 1 << 1,
+
+ // Mode: Names
+
+ /** An object had more than one field with the same name. */
+ DuplicateName = 1 << 2,
+ /** An object had a field with no name. */
+ MissingName = 1 << 3,
+ /** An array field had a name. */
+ ArrayName = 1 << 4,
+
+ // Mode: Format
+
+ /** A name or string payload is not valid UTF-8. */
+ InvalidString = 1 << 5,
+ /** A size or integer payload can be encoded in fewer bytes. */
+ InvalidInteger = 1 << 6,
+ /** A float64 payload can be encoded as a float32 without loss of precision. */
+ InvalidFloat = 1 << 7,
+ /** An object has the same type for every field but is not uniform. */
+ NonUniformObject = 1 << 8,
+ /** An array has the same type for every field and non-empty payloads but is not uniform. */
+ NonUniformArray = 1 << 9,
+
+ // Mode: Padding
+
+ /** A value did not use the entire memory view given for validation. */
+ Padding = 1 << 10,
+
+ // Mode: Package
+
+ /** The package or attachment had missing fields or fields out of order. */
+ InvalidPackageFormat = 1 << 11,
+ /** The object or an attachment did not match the hash stored for it. */
+ InvalidPackageHash = 1 << 12,
+ /** The package contained more than one copy of the same attachment. */
+ DuplicateAttachments = 1 << 13,
+ /** The package contained more than one object. */
+ MultiplePackageObjects = 1 << 14,
+ /** The package contained an object with no fields. */
+ NullPackageObject = 1 << 15,
+ /** The package contained a null attachment. */
+ NullPackageAttachment = 1 << 16,
+};
+
+ENUM_CLASS_FLAGS(CbValidateError);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Validate the compact binary data for one field in the view as specified by the mode flags.
+ *
+ * Only one top-level field is processed from the view, and validation recurses into any array or
+ * object within that field. To validate multiple consecutive top-level fields, call the function
+ * once for each top-level field. If the given view might contain multiple top-level fields, then
+ * either exclude the Padding flag from the Mode or use MeasureCompactBinary to break up the view
+ * into its constituent fields before validating.
+ *
+ * @param View A memory view containing at least one top-level field.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType);
+
+/**
+ * Validate the compact binary data for every field in the view as specified by the mode flags.
+ *
+ * This function expects the entire view to contain fields. Any trailing region of the view which
+ * does not contain a valid field will produce an OutOfBounds or InvalidType error instead of the
+ * Padding error that would be produced by the single field validation function.
+ *
+ * @see ValidateCompactBinary
+ */
+ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode);
+
+/**
+ * Validate the compact binary attachment pointed to by the view as specified by the mode flags.
+ *
+ * The attachment is validated with ValidateCompactBinary by using the validation mode specified.
+ * Include ECbValidateMode::Package to validate the attachment format and hash.
+ *
+ * @see ValidateCompactBinary
+ *
+ * @param View A memory view containing a package.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateCompactBinaryAttachment(MemoryView View, CbValidateMode Mode);
+
+/**
+ * Validate the compact binary package pointed to by the view as specified by the mode flags.
+ *
+ * The package, and attachments, are validated with ValidateCompactBinary by using the validation
+ * mode specified. Include ECbValidateMode::Package to validate the package format and hashes.
+ *
+ * @see ValidateCompactBinary
+ *
+ * @param View A memory view containing a package.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void usonvalidation_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h
new file mode 100644
index 000000000..759cf8444
--- /dev/null
+++ b/zencore/include/zencore/compress.h
@@ -0,0 +1,53 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore/zencore.h"
+
+namespace zen::CompressedBuffer {
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static constexpr uint64_t DefaultBlockSize = 256 * 1024;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Method used to compress the data in a compressed buffer. */
+enum class Method : uint8_t
+{
+ /** Header is followed by one uncompressed block. */
+ None = 0,
+ /** Header is followed by an array of compressed block sizes then the compressed blocks. */
+ LZ4 = 4,
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Header used on every compressed buffer. Always stored in big-endian format */
+struct BufferHeader
+{
+ static constexpr uint32_t ExpectedMagic = 0xb7756362;
+
+ /** A magic number to identify a compressed buffer. Always 0xb7756362 */
+ uint32_t Magic = ExpectedMagic;
+ /** A CRC-32 used to check integrity of the buffer. Uses the polynomial 0x04c11db7 */
+ uint32_t Crc32 = 0;
+ /** The method used to compress the buffer. Affects layout of data following the header */
+ Method Method = Method::None;
+ /** The reserved bytes must be initialized to zero */
+ uint8_t Reserved[2]{};
+ /** The power of two size of every uncompressed block except the last. Size is 1 << BlockSizeExponent */
+ uint8_t BlockSizeExponent = 0;
+ /** The number of blocks that follow the header */
+ uint32_t BlockCount = 0;
+ /** The total size of the uncompressed data */
+ uint64_t TotalRawSize = 0;
+ /** The total size of the compressed data including the header */
+ uint64_t TotalCompressedSize = 0;
+};
+
+static_assert(sizeof(BufferHeader) == 32, "BufferHeader is the wrong size");
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace zen::CompressedBuffer
diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h
new file mode 100644
index 000000000..27c831bb1
--- /dev/null
+++ b/zencore/include/zencore/endian.h
@@ -0,0 +1,61 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+namespace zen {
+
+inline uint16_t
+ByteSwap(uint16_t x)
+{
+ return _byteswap_ushort(x);
+}
+
+inline uint32_t
+ByteSwap(uint32_t x)
+{
+ return _byteswap_ulong(x);
+}
+
+inline uint64_t
+ByteSwap(uint64_t x)
+{
+ return _byteswap_uint64(x);
+}
+
+inline uint16_t
+FromNetworkOrder(uint16_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint32_t
+FromNetworkOrder(uint32_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint64_t
+FromNetworkOrder(uint64_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint16_t
+FromNetworkOrder(int16_t x)
+{
+ return ByteSwap(uint16_t(x));
+}
+
+inline uint32_t
+FromNetworkOrder(int32_t x)
+{
+ return ByteSwap(uint32_t(x));
+}
+
+inline uint64_t
+FromNetworkOrder(int64_t x)
+{
+ return ByteSwap(uint64_t(x));
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/enumflags.h b/zencore/include/zencore/enumflags.h
new file mode 100644
index 000000000..ebe747bf0
--- /dev/null
+++ b/zencore/include/zencore/enumflags.h
@@ -0,0 +1,61 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+// Enum class helpers
+
+// Defines all bitwise operators for enum classes so it can be (mostly) used as a regular flags enum
+#define ENUM_CLASS_FLAGS(Enum) \
+ inline Enum& operator|=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \
+ inline Enum& operator&=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \
+ inline Enum& operator^=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator|(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator&(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator^(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \
+ inline constexpr bool operator!(Enum E) { return !(__underlying_type(Enum))E; } \
+ inline constexpr Enum operator~(Enum E) { return (Enum) ~(__underlying_type(Enum))E; }
+
+// Friends all bitwise operators for enum classes so the definition can be kept private / protected.
+#define FRIEND_ENUM_CLASS_FLAGS(Enum) \
+ friend Enum& operator|=(Enum& Lhs, Enum Rhs); \
+ friend Enum& operator&=(Enum& Lhs, Enum Rhs); \
+ friend Enum& operator^=(Enum& Lhs, Enum Rhs); \
+ friend constexpr Enum operator|(Enum Lhs, Enum Rhs); \
+ friend constexpr Enum operator&(Enum Lhs, Enum Rhs); \
+ friend constexpr Enum operator^(Enum Lhs, Enum Rhs); \
+ friend constexpr bool operator!(Enum E); \
+ friend constexpr Enum operator~(Enum E);
+
+template<typename Enum>
+constexpr bool
+EnumHasAllFlags(Enum Flags, Enum Contains)
+{
+ return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) == ((__underlying_type(Enum))Contains);
+}
+
+template<typename Enum>
+constexpr bool
+EnumHasAnyFlags(Enum Flags, Enum Contains)
+{
+ return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) != 0;
+}
+
+template<typename Enum>
+void
+EnumAddFlags(Enum& Flags, Enum FlagsToAdd)
+{
+ Flags |= FlagsToAdd;
+}
+
+template<typename Enum>
+void
+EnumRemoveFlags(Enum& Flags, Enum FlagsToRemove)
+{
+ Flags &= ~FlagsToRemove;
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h
new file mode 100644
index 000000000..07c4833ff
--- /dev/null
+++ b/zencore/include/zencore/except.h
@@ -0,0 +1,60 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/string.h>
+#include <zencore/windows.h>
+#include <string>
+
+namespace zen {
+
+class WindowsException : public std::exception
+{
+public:
+ WindowsException(const char* Message)
+ {
+ m_hResult = HRESULT_FROM_WIN32(GetLastError());
+ m_Message = Message;
+ }
+
+ WindowsException(HRESULT hRes, const char* Message)
+ {
+ m_hResult = hRes;
+ m_Message = Message;
+ }
+
+ WindowsException(HRESULT hRes, const char* Message, const char* Detail)
+ {
+ m_hResult = hRes;
+
+ ExtendableStringBuilder<128> msg;
+ msg.Append(Message);
+ msg.Append(" (detail: '");
+ msg.Append(Detail);
+ msg.Append("')");
+
+ m_Message = msg.c_str();
+ }
+
+ virtual const char* what() const override { return m_Message.c_str(); }
+
+private:
+ std::string m_Message;
+ HRESULT m_hResult;
+};
+
+ZENCORE_API void ThrowSystemException(HRESULT hRes, const char* Message);
+inline void
+ThrowSystemException(const char* Message)
+{
+ throw WindowsException(Message);
+}
+
+inline void
+ThrowIfFailed(HRESULT hRes, const char* Message)
+{
+ if (FAILED(hRes))
+ ThrowSystemException(hRes, Message);
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
new file mode 100644
index 000000000..b20a1e7c6
--- /dev/null
+++ b/zencore/include/zencore/filesystem.h
@@ -0,0 +1,74 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "stream.h"
+#include "zencore.h"
+
+#include <filesystem>
+#include <functional>
+
+namespace zen {
+
+class IoBuffer;
+
+/** Delete directory (after deleting any contents)
+ */
+ZENCORE_API bool DeleteDirectories(const wchar_t* dir);
+ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir);
+
+/** Ensure directory exists.
+
+ Will also create any required parent directories
+ */
+ZENCORE_API bool CreateDirectories(const wchar_t* dir);
+ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir);
+
+/** Ensure directory exists and delete contents (if any) before returning
+ */
+ZENCORE_API bool CleanDirectory(const wchar_t* dir);
+ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir);
+
+struct FileContents
+{
+ std::vector<IoBuffer> Data;
+ std::error_code ErrorCode;
+};
+
+ZENCORE_API FileContents ReadFile(std::filesystem::path Path);
+ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc);
+ZENCORE_API bool WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount);
+
+struct CopyFileOptions
+{
+ bool EnableClone = true;
+ bool MustClone = false;
+};
+
+ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
+ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path);
+
+ZENCORE_API std::string ToUtf8(const std::filesystem::path& Path);
+
+/**
+ * Efficient file system traversal
+ *
+ * Uses the best available mechanism for the platform in question and could take
+ * advantage of any file system tracking mechanisms in the future
+ *
+ */
+class FileSystemTraversal
+{
+public:
+ struct TreeVisitor
+ {
+ virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) = 0;
+
+ // This should return true if we should recurse into the directory
+ virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) = 0;
+ };
+
+ void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
+};
+
+} // namespace zen
diff --git a/zencore/include/zencore/fmtutils.h b/zencore/include/zencore/fmtutils.h
new file mode 100644
index 000000000..fb5a08d56
--- /dev/null
+++ b/zencore/include/zencore/fmtutils.h
@@ -0,0 +1,49 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+
+#include <fmt/format.h>
+#include <filesystem>
+#include <string_view>
+
+// Custom formatting for some zencore types
+
+template<>
+struct fmt::formatter<zen::IoHash> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const zen::IoHash& Hash, FormatContext& ctx)
+ {
+ zen::IoHash::String_t String;
+ Hash.ToHexString(String);
+ return formatter<string_view>::format({String, zen::IoHash::StringLength}, ctx);
+ }
+};
+
+template<>
+struct fmt::formatter<zen::Oid> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const zen::Oid& Id, FormatContext& ctx)
+ {
+ zen::StringBuilder<32> String;
+ Id.ToString(String);
+ return formatter<string_view>::format({String.c_str(), zen::Oid::StringLength}, ctx);
+ }
+};
+
+template<>
+struct fmt::formatter<std::filesystem::path> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const std::filesystem::path& Path, FormatContext& ctx)
+ {
+ zen::ExtendableStringBuilder<128> String;
+ WideToUtf8(Path.c_str(), String);
+ return formatter<string_view>::format(String.ToView(), ctx);
+ }
+};
diff --git a/zencore/include/zencore/httpclient.h b/zencore/include/zencore/httpclient.h
new file mode 100644
index 000000000..4ede6839c
--- /dev/null
+++ b/zencore/include/zencore/httpclient.h
@@ -0,0 +1,18 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/string.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class HttpClient
+{
+};
+
+} // namespace zen
+
+void httpclient_forcelink(); // internal
diff --git a/zencore/include/zencore/httpserver.h b/zencore/include/zencore/httpserver.h
new file mode 100644
index 000000000..563245264
--- /dev/null
+++ b/zencore/include/zencore/httpserver.h
@@ -0,0 +1,373 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/enumflags.h>
+#include <zencore/refcount.h>
+#include <zencore/string.h>
+#include <functional>
+#include <gsl/gsl-lite.hpp>
+#include <list>
+#include <regex>
+#include <span>
+#include <unordered_map>
+#include "zencore.h"
+
+namespace zen {
+
+class IoBuffer;
+class CbObject;
+class StringBuilderBase;
+
+enum class HttpVerb
+{
+ kGet = 1 << 0,
+ kPut = 1 << 1,
+ kPost = 1 << 2,
+ kDelete = 1 << 3,
+ kHead = 1 << 4,
+ kCopy = 1 << 5,
+ kOptions = 1 << 6
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(HttpVerb);
+
+enum class HttpResponse
+{
+ // 1xx - Informational
+
+ Continue = 100, //!< Indicates that the initial part of a request has been received and has not yet been rejected by the server.
+ SwitchingProtocols = 101, //!< Indicates that the server understands and is willing to comply with the client's request, via the
+ //!< Upgrade header field, for a change in the application protocol being used on this connection.
+ Processing = 102, //!< Is an interim response used to inform the client that the server has accepted the complete request, but has not
+ //!< yet completed it.
+ EarlyHints = 103, //!< Indicates to the client that the server is likely to send a final response with the header fields included in
+ //!< the informational response.
+
+ // 2xx - Successful
+
+ OK = 200, //!< Indicates that the request has succeeded.
+ Created = 201, //!< Indicates that the request has been fulfilled and has resulted in one or more new resources being created.
+ Accepted = 202, //!< Indicates that the request has been accepted for processing, but the processing has not been completed.
+ NonAuthoritativeInformation = 203, //!< Indicates that the request was successful but the enclosed payload has been modified from that
+ //!< of the origin server's 200 (OK) response by a transforming proxy.
+ NoContent = 204, //!< Indicates that the server has successfully fulfilled the request and that there is no additional content to send
+ //!< in the response payload body.
+ ResetContent = 205, //!< Indicates that the server has fulfilled the request and desires that the user agent reset the \"document
+ //!< view\", which caused the request to be sent, to its original state as received from the origin server.
+ PartialContent = 206, //!< Indicates that the server is successfully fulfilling a range request for the target resource by transferring
+ //!< one or more parts of the selected representation that correspond to the satisfiable ranges found in the
+ //!< requests's Range header field.
+ MultiStatus = 207, //!< Provides status for multiple independent operations.
+ AlreadyReported = 208, //!< Used inside a DAV:propstat response element to avoid enumerating the internal members of multiple bindings
+ //!< to the same collection repeatedly. [RFC 5842]
+ IMUsed = 226, //!< The server has fulfilled a GET request for the resource, and the response is a representation of the result of one
+ //!< or more instance-manipulations applied to the current instance.
+
+ // 3xx - Redirection
+
+ MultipleChoices = 300, //!< Indicates that the target resource has more than one representation, each with its own more specific
+ //!< identifier, and information about the alternatives is being provided so that the user (or user agent) can
+ //!< select a preferred representation by redirecting its request to one or more of those identifiers.
+ MovedPermanently = 301, //!< Indicates that the target resource has been assigned a new permanent URI and any future references to this
+ //!< resource ought to use one of the enclosed URIs.
+ Found = 302, //!< Indicates that the target resource resides temporarily under a different URI.
+ SeeOther = 303, //!< Indicates that the server is redirecting the user agent to a different resource, as indicated by a URI in the
+ //!< Location header field, that is intended to provide an indirect response to the original request.
+ NotModified = 304, //!< Indicates that a conditional GET request has been received and would have resulted in a 200 (OK) response if it
+ //!< were not for the fact that the condition has evaluated to false.
+ UseProxy = 305, //!< \deprecated \parblock Due to security concerns regarding in-band configuration of a proxy. \endparblock
+ //!< The requested resource MUST be accessed through the proxy given by the Location field.
+ TemporaryRedirect = 307, //!< Indicates that the target resource resides temporarily under a different URI and the user agent MUST NOT
+ //!< change the request method if it performs an automatic redirection to that URI.
+ PermanentRedirect = 308, //!< The target resource has been assigned a new permanent URI and any future references to this resource
+ //!< ought to use one of the enclosed URIs. [...] This status code is similar to 301 Moved Permanently
+ //!< (Section 7.3.2 of rfc7231), except that it does not allow rewriting the request method from POST to GET.
+
+ // 4xx - Client Error
+ BadRequest = 400, //!< Indicates that the server cannot or will not process the request because the received syntax is invalid,
+ //!< nonsensical, or exceeds some limitation on what the server is willing to process.
+ Unauthorized = 401, //!< Indicates that the request has not been applied because it lacks valid authentication credentials for the
+ //!< target resource.
+ PaymentRequired = 402, //!< *Reserved*
+ Forbidden = 403, //!< Indicates that the server understood the request but refuses to authorize it.
+ NotFound = 404, //!< Indicates that the origin server did not find a current representation for the target resource or is not willing
+ //!< to disclose that one exists.
+ MethodNotAllowed = 405, //!< Indicates that the method specified in the request-line is known by the origin server but not supported by
+ //!< the target resource.
+ NotAcceptable = 406, //!< Indicates that the target resource does not have a current representation that would be acceptable to the
+ //!< user agent, according to the proactive negotiation header fields received in the request, and the server is
+ //!< unwilling to supply a default representation.
+ ProxyAuthenticationRequired =
+ 407, //!< Is similar to 401 (Unauthorized), but indicates that the client needs to authenticate itself in order to use a proxy.
+ RequestTimeout =
+ 408, //!< Indicates that the server did not receive a complete request message within the time that it was prepared to wait.
+ Conflict = 409, //!< Indicates that the request could not be completed due to a conflict with the current state of the resource.
+ Gone = 410, //!< Indicates that access to the target resource is no longer available at the origin server and that this condition is
+ //!< likely to be permanent.
+ LengthRequired = 411, //!< Indicates that the server refuses to accept the request without a defined Content-Length.
+ PreconditionFailed =
+ 412, //!< Indicates that one or more preconditions given in the request header fields evaluated to false when tested on the server.
+ PayloadTooLarge = 413, //!< Indicates that the server is refusing to process a request because the request payload is larger than the
+ //!< server is willing or able to process.
+ URITooLong = 414, //!< Indicates that the server is refusing to service the request because the request-target is longer than the
+ //!< server is willing to interpret.
+ UnsupportedMediaType = 415, //!< Indicates that the origin server is refusing to service the request because the payload is in a format
+ //!< not supported by the target resource for this method.
+ RangeNotSatisfiable = 416, //!< Indicates that none of the ranges in the request's Range header field overlap the current extent of the
+ //!< selected resource or that the set of ranges requested has been rejected due to invalid ranges or an
+ //!< excessive request of small or overlapping ranges.
+ ExpectationFailed = 417, //!< Indicates that the expectation given in the request's Expect header field could not be met by at least
+ //!< one of the inbound servers.
+ ImATeapot = 418, //!< Any attempt to brew coffee with a teapot should result in the error code 418 I'm a teapot.
+ UnprocessableEntity = 422, //!< Means the server understands the content type of the request entity (hence a 415(Unsupported Media
+ //!< Type) status code is inappropriate), and the syntax of the request entity is correct (thus a 400 (Bad
+ //!< Request) status code is inappropriate) but was unable to process the contained instructions.
+ Locked = 423, //!< Means the source or destination resource of a method is locked.
+ FailedDependency = 424, //!< Means that the method could not be performed on the resource because the requested action depended on
+ //!< another action and that action failed.
+ UpgradeRequired = 426, //!< Indicates that the server refuses to perform the request using the current protocol but might be willing to
+ //!< do so after the client upgrades to a different protocol.
+ PreconditionRequired = 428, //!< Indicates that the origin server requires the request to be conditional.
+ TooManyRequests = 429, //!< Indicates that the user has sent too many requests in a given amount of time (\"rate limiting\").
+ RequestHeaderFieldsTooLarge =
+ 431, //!< Indicates that the server is unwilling to process the request because its header fields are too large.
+ UnavailableForLegalReasons =
+ 451, //!< This status code indicates that the server is denying access to the resource in response to a legal demand.
+
+ // 5xx - Server Error
+
+ InternalServerError =
+ 500, //!< Indicates that the server encountered an unexpected condition that prevented it from fulfilling the request.
+ NotImplemented = 501, //!< Indicates that the server does not support the functionality required to fulfill the request.
+ BadGateway = 502, //!< Indicates that the server, while acting as a gateway or proxy, received an invalid response from an inbound
+ //!< server it accessed while attempting to fulfill the request.
+ ServiceUnavailable = 503, //!< Indicates that the server is currently unable to handle the request due to a temporary overload or
+ //!< scheduled maintenance, which will likely be alleviated after some delay.
+ GatewayTimeout = 504, //!< Indicates that the server, while acting as a gateway or proxy, did not receive a timely response from an
+ //!< upstream server it needed to access in order to complete the request.
+ HTTPVersionNotSupported = 505, //!< Indicates that the server does not support, or refuses to support, the protocol version that was
+ //!< used in the request message.
+ VariantAlsoNegotiates =
+ 506, //!< Indicates that the server has an internal configuration error: the chosen variant resource is configured to engage in
+ //!< transparent content negotiation itself, and is therefore not a proper end point in the negotiation process.
+ InsufficientStorage = 507, //!< Means the method could not be performed on the resource because the server is unable to store the
+ //!< representation needed to successfully complete the request.
+ LoopDetected = 508, //!< Indicates that the server terminated an operation because it encountered an infinite loop while processing a
+ //!< request with "Depth: infinity". [RFC 5842]
+ NotExtended = 510, //!< The policy for accessing the resource has not been met in the request. [RFC 2774]
+ NetworkAuthenticationRequired = 511, //!< Indicates that the client needs to authenticate to gain network access.
+};
+
+enum class HttpContentType
+{
+ kBinary,
+ kText,
+ kJSON,
+ kCbObject,
+ kCbPackage
+};
+
+/** HTTP Server Request
+ */
+class HttpServerRequest
+{
+public:
+ HttpServerRequest();
+ ~HttpServerRequest();
+
+ // Synchronous operations
+
+ inline [[nodiscard]] std::string_view RelativeUri() const { return m_Uri; } // Returns URI without service prefix
+ inline [[nodiscard]] std::string_view QueryString() const { return m_QueryString; }
+ inline bool IsHandled() const { return m_IsHandled; }
+
+ struct QueryParams
+ {
+ std::vector<std::pair<std::string_view, std::string_view>> KvPairs;
+
+ std::string_view GetValue(std::string_view ParamName)
+ {
+ for (const auto& Kv : KvPairs)
+ {
+ const std::string_view& Key = Kv.first;
+
+ if (Key.size() == ParamName.size())
+ {
+ if (0 == _strnicmp(Key.data(), ParamName.data(), Key.size()))
+ {
+ return Kv.second;
+ }
+ }
+ }
+
+ return std::string_view();
+ }
+ };
+
+ QueryParams GetQueryParams();
+
+ inline HttpVerb RequestVerb() const { return m_Verb; }
+
+ const char* HeaderAccept() const;
+ const char* HeaderAcceptEncoding() const;
+ const char* HeaderContentType() const;
+ const char* HeaderContentEncoding() const;
+ inline uint64_t HeaderContentLength() const { return m_ContentLength; }
+
+ void SetSuppressResponseBody() { m_SuppressBody = true; }
+
+ // Asynchronous operations
+
+ /** Read POST/PUT payload
+
+ This will return a null buffer if the contents are not fully available yet, and the handler should
+ at that point return - another completion request will be issues once the contents have been received
+ fully.
+ */
+ virtual IoBuffer ReadPayload() = 0;
+
+ /** Respond with payload
+
+ Note that this is destructive in the sense that the IoBuffer instances referred to by Blobs will be
+ moved into our response handler array where they are kept alive, in order to reduce ref-counting storms
+ */
+ virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) = 0;
+ virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, IoBuffer Blob);
+ virtual void WriteResponse(HttpResponse HttpResponseCode) = 0;
+
+ virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) = 0;
+
+ void WriteResponse(HttpResponse HttpResponseCode, CbObject Data);
+ void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::string_view ResponseString);
+
+protected:
+ bool m_IsHandled = false;
+ bool m_SuppressBody = false;
+ HttpVerb m_Verb = HttpVerb::kGet;
+ uint64_t m_ContentLength = ~0ull;
+ ExtendableStringBuilder<256> m_Uri;
+ ExtendableStringBuilder<256> m_QueryString;
+};
+
+class HttpServerException : public std::exception
+{
+public:
+ HttpServerException(const char* Message, uint32_t Error) : std::exception(Message), m_ErrorCode(Error) {}
+
+private:
+ uint32_t m_ErrorCode;
+};
+
+class HttpService
+{
+public:
+ HttpService() = default;
+ virtual ~HttpService() = default;
+
+ virtual const char* BaseUri() const = 0;
+ virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) = 0;
+
+ // Internals
+
+ inline void SetUriPrefixLength(size_t PrefixLength) { m_UriPrefixLength = (int)PrefixLength; }
+ inline int UriPrefixLength() const { return m_UriPrefixLength; }
+
+private:
+ int m_UriPrefixLength = 0;
+};
+
+/** HTTP server
+ */
+class HttpServer
+{
+public:
+ HttpServer();
+ ~HttpServer();
+
+ void AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler);
+ void AddEndpoint(HttpService& Service);
+
+ void Initialize(int BasePort);
+ void Run(bool TestMode);
+ void RequestExit();
+
+private:
+ struct Impl;
+
+ RefPtr<Impl> m_Impl;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+class HttpRouterRequest
+{
+public:
+ HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {}
+
+ ZENCORE_API std::string GetCapture(int Index) const;
+ inline HttpServerRequest& ServerRequest() { return m_HttpRequest; }
+
+private:
+ using MatchResults_t = std::match_results<std::string_view::const_iterator>;
+
+ HttpServerRequest& m_HttpRequest;
+ MatchResults_t m_Match;
+
+ friend class HttpRequestRouter;
+};
+
+inline std::string
+HttpRouterRequest::GetCapture(int Index) const
+{
+ ZEN_ASSERT(Index < m_Match.size());
+
+ return m_Match[Index];
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+class HttpRequestRouter
+{
+public:
+ typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t;
+
+ void AddPattern(const char* Id, const char* Regex);
+ void RegisterRoute(const char* Regex, HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs);
+ bool HandleRequest(zen::HttpServerRequest& Request);
+
+private:
+ struct HandlerEntry
+ {
+ HandlerEntry(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern)
+ : RegEx(Regex, std::regex::icase | std::regex::ECMAScript)
+ , Verbs(SupportedVerbs)
+ , Handler(std::move(Handler))
+ , Pattern(Pattern)
+ {
+ }
+
+ ~HandlerEntry() = default;
+
+ std::regex RegEx;
+ HttpVerb Verbs;
+ HandlerFunc_t Handler;
+ const char* Pattern;
+ };
+
+ std::list<HandlerEntry> m_Handlers;
+ std::unordered_map<std::string, std::string> m_PatternMap;
+};
+
+//////////////////////////////////////////////////////////////////////////
+//
+// HTTP Client
+//
+
+class HttpClient
+{
+};
+
+} // namespace zen
+
+void http_forcelink(); // internal
diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h
new file mode 100644
index 000000000..9d39b8226
--- /dev/null
+++ b/zencore/include/zencore/intmath.h
@@ -0,0 +1,140 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <stdint.h>
+#include <concepts>
+
+//////////////////////////////////////////////////////////////////////////
+
+#pragma intrinsic(_BitScanReverse)
+#pragma intrinsic(_BitScanReverse64)
+
+namespace zen {
+
+inline constexpr bool
+IsPow2(uint64_t n)
+{
+ return 0 == (n & (n - 1));
+}
+
+/// Round an integer up to the closest integer multiplier of 'base' ('base' must be a power of two)
+template<std::integral T>
+T
+RoundUp(T Value, auto Base)
+{
+ ZEN_ASSERT_SLOW(IsPow2(Base));
+ return ((Value + T(Base - 1)) & (~T(Base - 1)));
+}
+
+bool
+IsMultipleOf(std::integral auto Value, auto MultiplierPow2)
+{
+ ZEN_ASSERT_SLOW(IsPow2(MultiplierPow2));
+ return (Value & (MultiplierPow2 - 1)) == 0;
+}
+
+inline uint64_t
+NextPow2(uint64_t n)
+{
+ // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+
+ --n;
+
+ n |= n >> 1;
+ n |= n >> 2;
+ n |= n >> 4;
+ n |= n >> 8;
+ n |= n >> 16;
+ n |= n >> 32;
+
+ return n + 1;
+}
+
+static inline uint32_t
+FloorLog2(uint32_t Value)
+{
+ // Use BSR to return the log2 of the integer
+ unsigned long Log2;
+ if (_BitScanReverse(&Log2, Value) != 0)
+ {
+ return Log2;
+ }
+
+ return 0;
+}
+
+static inline uint32_t
+CountLeadingZeros(uint32_t Value)
+{
+ unsigned long Log2;
+ _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1);
+ return 32 - Log2;
+}
+
+static inline uint64_t
+FloorLog2_64(uint64_t Value)
+{
+ unsigned long Log2;
+ long Mask = -long(_BitScanReverse64(&Log2, Value) != 0);
+ return Log2 & Mask;
+}
+
+static inline uint64_t
+CountLeadingZeros64(uint64_t Value)
+{
+ unsigned long Log2;
+ long Mask = -long(_BitScanReverse64(&Log2, Value) != 0);
+ return ((63 - Log2) & Mask) | (64 & ~Mask);
+}
+
+static inline uint64_t
+CeilLogTwo64(uint64_t Arg)
+{
+ int64_t Bitmask = ((int64_t)(CountLeadingZeros64(Arg) << 57)) >> 63;
+ return (64 - CountLeadingZeros64(Arg - 1)) & (~Bitmask);
+}
+
+static inline uint64_t
+CountTrailingZeros64(uint64_t Value)
+{
+ if (Value == 0)
+ {
+ return 64;
+ }
+ unsigned long BitIndex; // 0-based, where the LSB is 0 and MSB is 31
+ _BitScanForward64(&BitIndex, Value); // Scans from LSB to MSB
+ return BitIndex;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+static inline bool
+IsPointerAligned(const void* Ptr, uint64_t Alignment)
+{
+ ZEN_ASSERT_SLOW(IsPow2(Alignment));
+
+ return 0 == (reinterpret_cast<uintptr_t>(Ptr) & (Alignment - 1));
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+#ifdef min
+# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead"
+#endif
+
+auto
+Min(auto x, auto y)
+{
+ return x < y ? x : y;
+}
+
+auto
+Max(auto x, auto y)
+{
+ return x > y ? x : y;
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
new file mode 100644
index 000000000..c93d27959
--- /dev/null
+++ b/zencore/include/zencore/iobuffer.h
@@ -0,0 +1,272 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <memory.h>
+#include "refcount.h"
+#include "zencore.h"
+
+namespace zen {
+
+struct IoBufferExtendedCore;
+
+struct IoBufferFileReference
+{
+ void* FileHandle;
+ uint64_t FileChunkOffset;
+ uint64_t FileChunkSize;
+};
+
+struct IoBufferCore
+{
+public:
+ IoBufferCore() = default;
+ inline IoBufferCore(const void* DataPtr, size_t SizeBytes) : m_DataPtr(DataPtr), m_DataBytes(SizeBytes) {}
+ inline IoBufferCore(const IoBufferCore* Outer, const void* DataPtr, size_t SizeBytes)
+ : m_DataPtr(DataPtr)
+ , m_DataBytes(SizeBytes)
+ , m_OuterCore(Outer)
+ {
+ }
+
+ ZENCORE_API explicit IoBufferCore(size_t SizeBytes);
+ ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment);
+ ZENCORE_API ~IoBufferCore();
+
+ // Reference counting
+
+ inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); }
+ inline uint32_t Release() const
+ {
+ const uint32_t NewRefCount = AtomicDecrement(const_cast<IoBufferCore*>(this)->m_RefCount);
+ if (NewRefCount == 0)
+ {
+ DeleteThis();
+ }
+ return NewRefCount;
+ }
+
+ // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it
+
+ IoBufferCore(const IoBufferCore&) = delete;
+ IoBufferCore(IoBufferCore&&) = delete;
+ IoBufferCore& operator=(const IoBufferCore&) = delete;
+ IoBufferCore& operator=(IoBufferCore&&) = delete;
+
+ //
+
+ ZENCORE_API void Materialize() const;
+ ZENCORE_API void DeleteThis() const;
+ ZENCORE_API void MakeOwned(bool Immutable = true);
+
+ inline void EnsureDataValid() const
+ {
+ if ((m_Flags & kIsExtended) && !(m_Flags & kIsMaterialized))
+ Materialize();
+ }
+
+ inline bool IsOwned() const { return !!(m_Flags & kIsOwned); }
+ inline bool IsImmutable() const { return !(m_Flags & kIsMutable); }
+ inline bool IsNull() const { return m_DataBytes == 0; }
+
+ inline IoBufferExtendedCore* ExtendedCore();
+ inline const IoBufferExtendedCore* ExtendedCore() const;
+
+ inline const void* DataPointer() const
+ {
+ EnsureDataValid();
+ return m_DataPtr;
+ }
+
+ inline size_t DataBytes() const { return m_DataBytes; }
+
+ inline void Set(const void* Ptr, size_t Sz)
+ {
+ m_DataPtr = Ptr;
+ m_DataBytes = Sz;
+ }
+
+ inline void SetIsOwned(bool NewState)
+ {
+ if (NewState)
+ {
+ m_Flags |= kIsOwned;
+ }
+ else
+ {
+ m_Flags &= ~kIsOwned;
+ }
+ }
+
+ inline void SetIsImmutable(bool NewState)
+ {
+ if (!NewState)
+ {
+ m_Flags |= kIsMutable;
+ }
+ else
+ {
+ m_Flags &= ~kIsMutable;
+ }
+ }
+
+ inline uint32_t GetRefCount() const { return m_RefCount; }
+
+protected:
+ uint32_t m_RefCount = 0;
+ mutable uint32_t m_Flags = 0;
+ mutable const void* m_DataPtr = nullptr;
+ size_t m_DataBytes = 0;
+ RefPtr<const IoBufferCore> m_OuterCore;
+
+ enum Flags
+ {
+ kIsOwned = 1 << 0,
+ kIsMutable = 1 << 1,
+ kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore
+ kIsMaterialized = 1 << 3, // Data pointers are valid
+ kLowLevelAlloc = 1 << 4, // Using direct memory allocation
+ };
+
+ void* AllocateBuffer(size_t InSize, size_t Alignment);
+ void FreeBuffer();
+};
+
+/**
+ * An "Extended" core references a segment of a file
+ */
+
+struct IoBufferExtendedCore : public IoBufferCore
+{
+ IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership);
+ IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size);
+ ~IoBufferExtendedCore();
+
+ enum ExtendedFlags
+ {
+ kOwnsFile = 1 << 8,
+ kOwnsMmap = 1 << 9
+ };
+
+ void Materialize() const;
+ bool GetFileReference(IoBufferFileReference& OutRef) const;
+
+private:
+ void* m_FileHandle = nullptr;
+ uint64_t m_FileOffset = 0;
+ mutable void* m_MmapHandle = nullptr;
+ mutable void* m_MappedPointer = nullptr;
+};
+
+inline IoBufferExtendedCore*
+IoBufferCore::ExtendedCore()
+{
+ if (m_Flags & kIsExtended)
+ {
+ return static_cast<IoBufferExtendedCore*>(this);
+ }
+
+ return nullptr;
+}
+
+inline const IoBufferExtendedCore*
+IoBufferCore::ExtendedCore() const
+{
+ if (m_Flags & kIsExtended)
+ {
+ return static_cast<const IoBufferExtendedCore*>(this);
+ }
+
+ return nullptr;
+}
+
+/**
+ * I/O buffer
+ *
+ * This represents a reference to a payload in memory or on disk
+ *
+ */
+class IoBuffer
+{
+public:
+ enum EAssumeOwnershipTag
+ {
+ AssumeOwnership
+ };
+ enum ECloneTag
+ {
+ Clone
+ };
+ enum EWrapTag
+ {
+ Wrap
+ };
+ enum EFileTag
+ {
+ File
+ };
+ enum EBorrowedFileTag
+ {
+ BorrowedFile
+ };
+
+ inline IoBuffer() = default;
+ inline IoBuffer(IoBuffer&& Rhs) noexcept = default;
+ inline IoBuffer(const IoBuffer& Rhs) = default;
+ inline IoBuffer& operator=(const IoBuffer& Rhs) = default;
+ inline IoBuffer& operator=(IoBuffer&& Rhs) noexcept = default;
+
+ /** Create an uninitialized buffer of the given size
+ */
+ ZENCORE_API explicit IoBuffer(size_t InSize);
+
+ /** Create an uninitialized buffer of the given size with the specified alignment
+ */
+ ZENCORE_API explicit IoBuffer(size_t InSize, uint64_t InAlignment);
+
+ /** Create a buffer which references a sequence of bytes inside another buffer
+ */
+ ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes);
+
+ /** Create a buffer which references a range of bytes which we assume will live
+ * for the entire life time.
+ */
+ inline IoBuffer(EWrapTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(DataPtr, SizeBytes)) {}
+
+ inline IoBuffer(ECloneTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(SizeBytes))
+ {
+ memcpy(const_cast<void*>(m_Core->DataPointer()), DataPtr, SizeBytes);
+ }
+
+ inline IoBuffer(EAssumeOwnershipTag, const void* DataPtr, size_t Sz) : m_Core(new IoBufferCore(DataPtr, Sz))
+ {
+ m_Core->SetIsOwned(true);
+ }
+
+ ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize);
+ ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize);
+
+ inline operator bool() const { return !m_Core->IsNull(); }
+ ZENCORE_API void MakeOwned() { return m_Core->MakeOwned(); }
+ inline bool IsOwned() const { return m_Core->IsOwned(); }
+ const void* Data() const { return m_Core->DataPointer(); }
+ const size_t Size() const { return m_Core->DataBytes(); }
+ ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const;
+
+private:
+ RefPtr<IoBufferCore> m_Core = new IoBufferCore;
+};
+
+class IoBufferBuilder
+{
+public:
+ ZENCORE_API static IoBuffer MakeFromFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
+ ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull);
+ inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
+
+private:
+};
+
+void iobuffer_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/iohash.h b/zencore/include/zencore/iohash.h
new file mode 100644
index 000000000..4eac7e328
--- /dev/null
+++ b/zencore/include/zencore/iohash.h
@@ -0,0 +1,95 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/blake3.h>
+#include <zencore/memory.h>
+
+#include <string_view>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/**
+ * Hash used for content addressable storage
+ *
+ * This is basically a BLAKE3-160 hash (note: this is probably not an officially
+ * recognized identifier). It is generated by computing a 32-byte BLAKE3 hash and
+ * picking the first 20 bytes of the resulting hash.
+ *
+ */
+struct IoHash
+{
+ uint8_t Hash[20];
+
+ static IoHash MakeFrom(const void* data /* 20 bytes */)
+ {
+ IoHash Io;
+ memcpy(Io.Hash, data, sizeof Io);
+ return Io;
+ }
+
+ static IoHash HashMemory(const void* data, size_t byteCount);
+ static IoHash HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); }
+ static IoHash FromHexString(const char* string);
+ static IoHash FromHexString(const std::string_view string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+ std::string ToHexString() const;
+
+ static const int StringLength = 40;
+ typedef char String_t[StringLength + 1];
+
+ static IoHash Zero; // Initialized to all zeros
+
+ inline auto operator<=>(const IoHash& rhs) const = default;
+
+ struct Hasher
+ {
+ size_t operator()(const IoHash& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct IoHashStream
+{
+ /// Begin streaming hash compute (not needed on freshly constructed instance)
+ void Reset() { m_Blake3Stream.Reset(); }
+
+ /// Append another chunk
+ IoHashStream& Append(const void* data, size_t byteCount)
+ {
+ m_Blake3Stream.Append(data, byteCount);
+ return *this;
+ }
+
+ /// Append another chunk
+ IoHashStream& Append(MemoryView Data)
+ {
+ m_Blake3Stream.Append(Data.GetData(), Data.GetSize());
+ return *this;
+ }
+
+ /// Obtain final hash. If you wish to reuse the instance call reset()
+ IoHash GetHash()
+ {
+ BLAKE3 b3 = m_Blake3Stream.GetHash();
+
+ IoHash Io;
+ memcpy(Io.Hash, b3.Hash, sizeof Io.Hash);
+
+ return Io;
+ }
+
+private:
+ BLAKE3Stream m_Blake3Stream;
+};
+
+} // namespace zen
diff --git a/zencore/include/zencore/md5.h b/zencore/include/zencore/md5.h
new file mode 100644
index 000000000..4ed4f6c56
--- /dev/null
+++ b/zencore/include/zencore/md5.h
@@ -0,0 +1,50 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <stdint.h>
+#include <compare>
+#include "zencore.h"
+
+namespace zen {
+
+class StringBuilderBase;
+
+struct MD5
+{
+ uint8_t Hash[16];
+
+ inline auto operator<=>(const MD5& rhs) const = default;
+
+ static const int StringLength = 40;
+ typedef char String_t[StringLength + 1];
+
+ static MD5 HashMemory(const void* data, size_t byteCount);
+ static MD5 FromHexString(const char* string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static MD5 Zero; // Initialized to all zeroes
+};
+
+/**
+ * Utility class for computing SHA1 hashes
+ */
+class MD5Stream
+{
+public:
+ MD5Stream();
+
+ /// Begin streaming MD5 compute (not needed on freshly constructed MD5Stream instance)
+ void Reset();
+ /// Append another chunk
+ MD5Stream& Append(const void* data, size_t byteCount);
+ /// Obtain final MD5 hash. If you wish to reuse the MD5Stream instance call reset()
+ MD5 GetHash();
+
+private:
+};
+
+void md5_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h
new file mode 100644
index 000000000..8a16126ef
--- /dev/null
+++ b/zencore/include/zencore/memory.h
@@ -0,0 +1,213 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/thread.h>
+
+#include <algorithm>
+#include <vector>
+
+namespace zen {
+
+class MemoryArena
+{
+public:
+ ZENCORE_API MemoryArena();
+ ZENCORE_API ~MemoryArena();
+
+ ZENCORE_API void* Alloc(size_t size, size_t alignment);
+ ZENCORE_API void Free(void* ptr);
+
+private:
+};
+
+class Memory
+{
+public:
+ ZENCORE_API static void* Alloc(size_t size, size_t alignment = sizeof(void*));
+ ZENCORE_API static void Free(void* ptr);
+};
+
+/** Allocator which claims fixed-size blocks from the underlying allocator.
+
+ There is no way to free individual memory blocks.
+
+ \note This is not thread-safe, you will need to provide synchronization yourself
+*/
+
+class ChunkingLinearAllocator
+{
+public:
+ ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment = sizeof(std::max_align_t));
+ ~ChunkingLinearAllocator();
+
+ ZENCORE_API void Reset();
+
+ ZENCORE_API void* Alloc(size_t Size, size_t Alignment = sizeof(void*));
+ inline void Free(void* Ptr) { ZEN_UNUSED(Ptr); /* no-op */ }
+
+private:
+ uint8_t* m_ChunkCursor = nullptr;
+ uint64_t m_ChunkBytesRemain = 0;
+ const uint64_t m_ChunkSize = 0;
+ const uint64_t m_ChunkAlignment = 0;
+ std::vector<void*> m_ChunkList;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct MutableMemoryView
+{
+ MutableMemoryView() = default;
+
+ MutableMemoryView(void* DataPtr, size_t DataSize)
+ : m_Data(reinterpret_cast<uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<uint8_t*>(DataPtr) + DataSize)
+ {
+ }
+
+ inline bool IsEmpty() const { return m_Data == m_DataEnd; }
+ void* GetData() const { return m_Data; }
+ void* GetDataEnd() const { return m_DataEnd; }
+ size_t GetSize() const { return reinterpret_cast<uint8_t*>(m_DataEnd) - reinterpret_cast<uint8_t*>(m_Data); }
+
+ inline bool EqualBytes(const MutableMemoryView& InView) const
+ {
+ const size_t Size = GetSize();
+
+ return Size == InView.GetSize() && (memcmp(m_Data, InView.m_Data, Size) == 0);
+ }
+
+ /** Modifies the view by chopping the given number of bytes from the left. */
+ inline void RightChopInline(uint64_t InSize)
+ {
+ const uint64_t Offset = std::min(GetSize(), InSize);
+ m_Data = GetDataAtOffsetNoCheck(Offset);
+ }
+
+ /** Returns the left-most part of the view by taking the given number of bytes from the left. */
+ constexpr inline MutableMemoryView Left(uint64_t InSize) const
+ {
+ MutableMemoryView View(*this);
+ View.LeftInline(InSize);
+ return View;
+ }
+
+ /** Modifies the view to be the given number of bytes from the left. */
+ constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = std::min(m_DataEnd, m_Data + InSize); }
+
+private:
+ uint8_t* m_Data = nullptr;
+ uint8_t* m_DataEnd = nullptr;
+
+ /** Returns the data pointer advanced by an offset in bytes. */
+ inline uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; }
+};
+
+struct MemoryView
+{
+ MemoryView() = default;
+
+ MemoryView(const MutableMemoryView& MutableView)
+ : m_Data(reinterpret_cast<const uint8_t*>(MutableView.GetData()))
+ , m_DataEnd(m_Data + MutableView.GetSize())
+ {
+ }
+
+ MemoryView(const void* DataPtr, size_t DataSize)
+ : m_Data(reinterpret_cast<const uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<const uint8_t*>(DataPtr) + DataSize)
+ {
+ }
+
+ MemoryView(const void* DataPtr, const void* DataEndPtr)
+ : m_Data(reinterpret_cast<const uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<const uint8_t*>(DataEndPtr))
+ {
+ }
+
+ inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); }
+ inline bool IsEmpty() const { return m_Data == m_DataEnd; }
+ const void* GetData() const { return m_Data; }
+ const void* GetDataEnd() const { return m_DataEnd; }
+ size_t GetSize() const { return reinterpret_cast<const uint8_t*>(m_DataEnd) - reinterpret_cast<const uint8_t*>(m_Data); }
+ inline bool operator==(const MemoryView& Rhs) const { return m_Data == Rhs.m_Data && m_DataEnd == Rhs.m_DataEnd; }
+
+ inline bool EqualBytes(const MemoryView& InView) const
+ {
+ const size_t Size = GetSize();
+
+ return Size == InView.GetSize() && (memcmp(m_Data, InView.GetData(), Size) == 0);
+ }
+
+ inline MemoryView& operator+=(size_t InSize)
+ {
+ RightChopInline(InSize);
+ return *this;
+ }
+
+ /** Modifies the view by chopping the given number of bytes from the left. */
+ inline void RightChopInline(uint64_t InSize)
+ {
+ const uint64_t Offset = std::min(GetSize(), InSize);
+ m_Data = GetDataAtOffsetNoCheck(Offset);
+ }
+
+ inline MemoryView RightChop(uint64_t InSize)
+ {
+ MemoryView View(*this);
+ View.RightChopInline(InSize);
+ return View;
+ }
+
+ /** Returns the left-most part of the view by taking the given number of bytes from the left. */
+ constexpr inline MemoryView Left(uint64_t InSize) const
+ {
+ MemoryView View(*this);
+ View.LeftInline(InSize);
+ return View;
+ }
+
+ /** Modifies the view to be the given number of bytes from the left. */
+ constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = std::min(m_DataEnd, m_Data + InSize); }
+
+ constexpr void Reset()
+ {
+ m_Data = nullptr;
+ m_DataEnd = nullptr;
+ }
+
+private:
+ const uint8_t* m_Data = nullptr;
+ const uint8_t* m_DataEnd = nullptr;
+
+ /** Returns the data pointer advanced by an offset in bytes. */
+ inline const uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; }
+};
+
+/**
+ * Make a non-owning view of the memory of the initializer list.
+ *
+ * This overload is only available when the element type does not need to be deduced.
+ */
+template<typename T>
+[[nodiscard]] inline MemoryView
+MakeMemoryView(std::initializer_list<typename std::type_identity<T>::type> List)
+{
+ return MemoryView(List.begin(), List.size() * sizeof(T));
+}
+
+/** Make a non-owning view of the memory of the contiguous container. */
+template<std::ranges::contiguous_range R>
+[[nodiscard]] constexpr inline MemoryView
+MakeMemoryView(const R& Container)
+{
+ const auto& Front = *std::begin(Container);
+ return MemoryView(std::addressof(Front), std::size(Container) * sizeof(Front));
+}
+
+void memory_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/meta.h b/zencore/include/zencore/meta.h
new file mode 100644
index 000000000..82eb5cc30
--- /dev/null
+++ b/zencore/include/zencore/meta.h
@@ -0,0 +1,30 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/* This file contains utility functions for meta programming
+ *
+ * Since you're in here you're probably quite observant, and you'll
+ * note that it's quite barren here. This is because template
+ * metaprogramming is awful and I try not to engage in it. However,
+ * sometimes these things are forced upon us.
+ *
+ */
+
+namespace zen {
+
+/**
+ * Uses implicit conversion to create an instance of a specific type.
+ * Useful to make things clearer or circumvent unintended type deduction in templates.
+ * Safer than C casts and static_casts, e.g. does not allow down-casts
+ *
+ * @param Obj The object (usually pointer or reference) to convert.
+ *
+ * @return The object converted to the specified type.
+ */
+template<typename T>
+inline T
+ImplicitConv(typename std::type_identity<T>::type Obj)
+{
+ return Obj;
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h
new file mode 100644
index 000000000..de7d315f4
--- /dev/null
+++ b/zencore/include/zencore/refcount.h
@@ -0,0 +1,144 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+#pragma once
+
+#include "atomic.h"
+#include "zencore.h"
+
+namespace zen {
+
+/**
+ * Helper base class for reference counted objects using intrusive reference counts
+ *
+ * This class is pretty straightforward but does one thing which may be unexpected:
+ *
+ * - Instances on the stack are initialized with a reference count of one to ensure
+ * nobody tries to accidentally delete it. (TODO: is this really useful?)
+ */
+class RefCounted
+{
+public:
+ RefCounted() : m_RefCount(IsPointerToStack(this)){};
+ virtual ~RefCounted() = default;
+
+ inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); }
+ inline uint32_t Release() const
+ {
+ uint32_t refCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount);
+ if (refCount == 0)
+ {
+ delete this;
+ }
+ return refCount;
+ }
+
+ // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it
+
+ RefCounted(const RefCounted&) = delete;
+ RefCounted(RefCounted&&) = delete;
+ RefCounted& operator=(const RefCounted&) = delete;
+ RefCounted& operator=(RefCounted&&) = delete;
+
+protected:
+ inline uint32_t RefCount() const { return m_RefCount; }
+
+private:
+ uint32_t m_RefCount = 0;
+};
+
+/**
+ * Smart pointer for classes derived from RefCounted
+ */
+
+template<class T>
+class RefPtr
+{
+public:
+ inline RefPtr() = default;
+ inline RefPtr(const RefPtr& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); }
+ inline RefPtr(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); }
+ inline ~RefPtr() { m_Ref && m_Ref->Release(); }
+
+ inline explicit operator bool() const { return m_Ref != nullptr; }
+ inline operator T*() const { return m_Ref; }
+ inline T* operator->() const { return m_Ref; }
+
+ inline std::strong_ordering operator<=>(const RefPtr& Rhs) const = default;
+
+ inline RefPtr& operator=(T* Rhs)
+ {
+ Rhs && Rhs->AddRef();
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs;
+ return *this;
+ }
+ inline RefPtr& operator=(const RefPtr& Rhs)
+ {
+ m_Ref && m_Ref->Release();
+ auto Ref = m_Ref = Rhs.m_Ref;
+ Ref && Ref->AddRef();
+ return *this;
+ }
+ inline RefPtr& operator=(RefPtr&& Rhs) noexcept
+ {
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ Rhs.m_Ref = nullptr;
+ return *this;
+ }
+ inline RefPtr(RefPtr&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
+
+private:
+ T* m_Ref = nullptr;
+};
+
+/**
+ * Smart pointer for classes derived from RefCounted
+ *
+ * This variant does not decay to a raw pointer
+ *
+ */
+
+template<class T>
+class Ref
+{
+public:
+ inline Ref() = default;
+ inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); }
+ inline Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); }
+ inline ~Ref() { m_Ref && m_Ref->Release(); }
+
+ inline explicit operator bool() const { return m_Ref != nullptr; }
+ inline T* operator->() const { return m_Ref; }
+
+ inline std::strong_ordering operator<=>(const Ref& Rhs) const = default;
+
+ inline Ref& operator=(T* Rhs)
+ {
+ Rhs && Rhs->AddRef();
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs;
+ return *this;
+ }
+ inline Ref& operator=(const Ref& Rhs)
+ {
+ m_Ref && m_Ref->Release();
+ auto Ref = m_Ref = Rhs.m_Ref;
+ Ref && Ref->AddRef();
+ return *this;
+ }
+ inline Ref& operator=(Ref&& Rhs) noexcept
+ {
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ Rhs.m_Ref = nullptr;
+ return *this;
+ }
+ inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
+
+private:
+ T* m_Ref = nullptr;
+};
+
+void refcount_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/scopeguard.h b/zencore/include/zencore/scopeguard.h
new file mode 100644
index 000000000..ba8cd3094
--- /dev/null
+++ b/zencore/include/zencore/scopeguard.h
@@ -0,0 +1,33 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <type_traits>
+#include "zencore.h"
+
+namespace zen {
+
+template<typename T>
+class ScopeGuardImpl
+{
+public:
+ inline ScopeGuardImpl(T&& func) : m_guardFunc(func) {}
+ ~ScopeGuardImpl()
+ {
+ if (!m_dismissed)
+ m_guardFunc();
+ }
+
+ void Dismiss() { m_dismissed = true; }
+
+private:
+ bool m_dismissed = false;
+ T m_guardFunc;
+};
+
+template<typename T>
+ScopeGuardImpl<T>
+MakeGuard(T&& fn)
+{
+ return ScopeGuardImpl<T>(std::move(fn));
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/sha1.h b/zencore/include/zencore/sha1.h
new file mode 100644
index 000000000..fc26f442b
--- /dev/null
+++ b/zencore/include/zencore/sha1.h
@@ -0,0 +1,76 @@
+// //////////////////////////////////////////////////////////
+// sha1.h
+// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved.
+// see http://create.stephan-brumme.com/disclaimer.html
+//
+
+#pragma once
+
+#include <stdint.h>
+#include <compare>
+#include "zencore.h"
+
+namespace zen {
+
+class StringBuilderBase;
+
+struct SHA1
+{
+ uint8_t Hash[20];
+
+ inline auto operator<=>(const SHA1& rhs) const = default;
+
+ static const int StringLength = 40;
+ typedef char String_t[StringLength + 1];
+
+ static SHA1 HashMemory(const void* data, size_t byteCount);
+ static SHA1 FromHexString(const char* string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static SHA1 Zero; // Initialized to all zeroes
+};
+
+/**
+ * Utility class for computing SHA1 hashes
+ */
+class SHA1Stream
+{
+public:
+ SHA1Stream();
+
+ /** compute SHA1 of a memory block
+
+ \note SHA1 class contains a slightly more convenient helper function for this use case
+ \see SHA1::fromMemory()
+ */
+ SHA1 Compute(const void* data, size_t byteCount);
+
+ /// Begin streaming SHA1 compute (not needed on freshly constructed SHA1Stream instance)
+ void Reset();
+ /// Append another chunk
+ SHA1Stream& Append(const void* data, size_t byteCount);
+ /// Obtain final SHA1 hash. If you wish to reuse the SHA1Stream instance call reset()
+ SHA1 GetHash();
+
+private:
+ void ProcessBlock(const void* data);
+ void ProcessBuffer();
+
+ enum
+ {
+ /// split into 64 byte blocks (=> 512 bits)
+ BlockSize = 512 / 8,
+ HashBytes = 20,
+ HashValues = HashBytes / 4
+ };
+
+ uint64_t m_NumBytes; // size of processed data in bytes
+ size_t m_BufferSize; // valid bytes in m_buffer
+ uint8_t m_Buffer[BlockSize]; // bytes not processed yet
+ uint32_t m_Hash[HashValues];
+};
+
+void sha1_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h
new file mode 100644
index 000000000..c6206f780
--- /dev/null
+++ b/zencore/include/zencore/sharedbuffer.h
@@ -0,0 +1,169 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/memory.h>
+#include <zencore/refcount.h>
+
+#include <memory.h>
+
+namespace zen {
+
+class BufferOwner : public RefCounted
+{
+protected:
+ inline BufferOwner(void* DataPtr, uint64_t DataSize, bool IsOwned, BufferOwner* OuterBuffer = nullptr)
+ : m_IsOwned(IsOwned)
+ , m_Data(DataPtr)
+ , m_Size(DataSize)
+ , m_Outer(OuterBuffer)
+ {
+ }
+
+ virtual ~BufferOwner();
+
+ // Ownership is a transitive property, and m_IsOwned currently only flags that this instance is responsible
+ // for managing the allocated memory, so we need to make recursive calls. Could be optimized slightly by
+ // adding a dedicated flag
+ inline bool IsOwned() const
+ {
+ if (m_IsOwned)
+ {
+ return true;
+ }
+ else
+ {
+ return m_Outer && m_Outer->IsOwned();
+ }
+ }
+
+ BufferOwner(const BufferOwner&) = delete;
+ BufferOwner& operator=(const BufferOwner&) = delete;
+
+private:
+ bool m_IsOwned;
+ void* m_Data;
+ uint64_t m_Size;
+ RefPtr<BufferOwner> m_Outer;
+
+ friend class UniqueBuffer;
+ friend class SharedBuffer;
+};
+
+/**
+ * Reference to a memory buffer with a single owner (see std::unique_ptr)
+ */
+class UniqueBuffer
+{
+public:
+ UniqueBuffer(const UniqueBuffer&) = delete;
+ UniqueBuffer& operator=(const UniqueBuffer&) = delete;
+
+ UniqueBuffer() = default;
+ ZENCORE_API explicit UniqueBuffer(BufferOwner* Owner);
+
+ void* GetData() { return m_buffer->m_Data; }
+ const void* GetData() const { return m_buffer->m_Data; }
+ size_t GetSize() const { return m_buffer->m_Size; }
+ operator MutableMemoryView() { return GetView(); }
+ operator MemoryView() const { return MemoryView(m_buffer->m_Data, m_buffer->m_Size); }
+
+ MutableMemoryView GetView() { return MutableMemoryView(m_buffer->m_Data, m_buffer->m_Size); }
+
+ /** Make an uninitialized owned buffer of the specified size. */
+ ZENCORE_API static UniqueBuffer Alloc(uint64_t Size);
+
+ /** Make a non-owned view of the input. */
+ ZENCORE_API static UniqueBuffer MakeView(void* DataPtr, uint64_t Size);
+
+private:
+ RefPtr<BufferOwner> m_buffer;
+
+ friend class SharedBuffer;
+};
+
+/**
+ * Reference to a memory buffer with shared ownership
+ */
+class SharedBuffer
+{
+public:
+ SharedBuffer() = default;
+ ZENCORE_API explicit SharedBuffer(UniqueBuffer&&);
+ inline explicit SharedBuffer(BufferOwner* Owner) : m_buffer(Owner) {}
+
+ void* GetData()
+ {
+ if (m_buffer)
+ {
+ return m_buffer->m_Data;
+ }
+ return nullptr;
+ }
+
+ const void* GetData() const
+ {
+ if (m_buffer)
+ {
+ return m_buffer->m_Data;
+ }
+ return nullptr;
+ }
+
+ size_t GetSize() const
+ {
+ if (m_buffer)
+ {
+ return m_buffer->m_Size;
+ }
+ return 0;
+ }
+
+ ZENCORE_API void MakeOwned();
+ bool IsOwned() const { return m_buffer && m_buffer->IsOwned(); }
+ inline explicit operator bool() const { return m_buffer; }
+ inline bool IsNull() const { return !m_buffer; }
+ inline void Reset() { m_buffer = nullptr; }
+
+ MemoryView GetView() const
+ {
+ if (m_buffer)
+ {
+ return MemoryView(m_buffer->m_Data, m_buffer->m_Size);
+ }
+ else
+ {
+ return MemoryView();
+ }
+ }
+
+ operator MemoryView() const { return GetView(); }
+
+ SharedBuffer& operator=(UniqueBuffer&& Rhs)
+ {
+ m_buffer = std::move(Rhs.m_buffer);
+ return *this;
+ }
+
+ std::strong_ordering operator<=>(const SharedBuffer& Rhs) const = default;
+
+ /** Make a non-owned view of the input */
+ inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); }
+ /** Make a non-owned view of the input */
+ ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size);
+ /** Make a non-owned view of the input */
+ ZENCORE_API static SharedBuffer MakeView(MemoryView View, SharedBuffer Buffer);
+ /** Make am owned clone of the buffer */
+ ZENCORE_API SharedBuffer Clone();
+ /** Make an owned clone of the memory in the input view */
+ ZENCORE_API static SharedBuffer Clone(MemoryView View);
+
+private:
+ RefPtr<BufferOwner> m_buffer;
+};
+
+void sharedbuffer_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/snapshot_manifest.h b/zencore/include/zencore/snapshot_manifest.h
new file mode 100644
index 000000000..95e64773a
--- /dev/null
+++ b/zencore/include/zencore/snapshot_manifest.h
@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/zencore.h>
+
+#include <filesystem>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace zen {
+
+struct LeafNode
+{
+ uint64_t FileSize = 0;
+ uint64_t FileModifiedTime = 0;
+ zen::IoHash ChunkHash = zen::IoHash::Zero;
+ std::wstring Name;
+};
+
+struct TreeNode
+{
+ std::vector<TreeNode> Children;
+ std::vector<LeafNode> Leaves;
+ std::wstring Name;
+ zen::BLAKE3 ChunkHash = zen::BLAKE3::Zero;
+
+ ZENCORE_API void VisitModifyFiles(std::function<void(LeafNode& node)> func);
+ ZENCORE_API void VisitFiles(std::function<void(const LeafNode& node)> func);
+ ZENCORE_API void Finalize();
+};
+
+struct SnapshotManifest
+{
+ std::string Id;
+ TreeNode Root;
+ zen::BLAKE3 ChunkHash = zen::BLAKE3::Zero;
+
+ ZENCORE_API void finalize();
+};
+
+class InStream;
+class OutStream;
+
+ZENCORE_API void ReadManifest(SnapshotManifest& Manifest, InStream& FromStream);
+ZENCORE_API void WriteManifest(const SnapshotManifest& Manifest, OutStream& ToStream);
+ZENCORE_API void PrintManifest(const SnapshotManifest& Manifest, OutStream& ToStream);
+
+// Translate a user-provided manifest specification into a file path.
+// Supports hashtag syntax to implicitly refer to user documents zenfs folder
+ZENCORE_API std::filesystem::path ManifestSpecToPath(const char* ManifestSpec);
+
+void snapshotmanifest_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h
new file mode 100644
index 000000000..7290fd914
--- /dev/null
+++ b/zencore/include/zencore/stats.h
@@ -0,0 +1,66 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <atomic>
+#include <type_traits>
+#include "zencore.h"
+
+namespace zen {
+
+template<typename T>
+class Gauge
+{
+public:
+ Gauge() : m_value{0} {}
+
+private:
+ T m_value;
+};
+
+class Counter
+{
+public:
+ inline void SetValue(uint64_t Value) { m_count = Value; }
+ inline uint64_t Value() const { return m_count; }
+
+ inline void Increment(int64_t AddValue) { m_count += AddValue; }
+ inline void Decrement(int64_t SubValue) { m_count -= SubValue; }
+ inline void Clear() { m_count = 0; }
+
+private:
+ std::atomic_uint64_t m_count{0};
+};
+
+/// <summary>
+/// Exponential Weighted Moving Average
+/// </summary>
+class EWMA
+{
+public:
+ /// <summary>
+ /// Update EWMA with new measure
+ /// </summary>
+ /// <param name="Alpha">Smoothing factor (between 0 and 1)</param>
+ /// <param name="Interval">Elapsed time since last</param>
+ /// <param name="Count">Value</param>
+ /// <param name="IsInitialUpdate">Whether this is the first update or not</param>
+ void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate);
+ double Rate() const;
+
+private:
+ double m_rate = 0;
+};
+
+/// <summary>
+/// Tracks rate of events over time (i.e requests/sec)
+/// </summary>
+class Meter
+{
+public:
+private:
+};
+
+extern void stats_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/stream.h b/zencore/include/zencore/stream.h
new file mode 100644
index 000000000..4e8c58382
--- /dev/null
+++ b/zencore/include/zencore/stream.h
@@ -0,0 +1,318 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/memory.h>
+#include <zencore/refcount.h>
+#include <zencore/thread.h>
+
+#include <string_view>
+#include <vector>
+
+namespace zen {
+
+/**
+ * Basic byte stream interface
+ *
+ * This is intended as a minimal base class offering only the absolute minimum of functionality.
+ *
+ * IMPORTANT: To better support concurrency, this abstraction offers no "file pointer". Thus
+ * every read or write operation needs to specify the offset from which they wish to read.
+ *
+ * Most client code will likely want to use reader/writer classes like BinaryWriter/BinaryReader
+ *
+ */
+class OutStream : public RefCounted
+{
+public:
+ virtual void Write(const void* Data, size_t ByteCount, uint64_t Offset) = 0;
+ virtual void Flush() = 0;
+};
+
+class InStream : public RefCounted
+{
+public:
+ virtual void Read(void* DataPtr, size_t ByteCount, uint64_t Offset) = 0;
+ virtual uint64_t Size() const = 0;
+};
+
+/**
+ * Stream which writes into a growing memory buffer
+ */
+class MemoryOutStream : public OutStream
+{
+public:
+ MemoryOutStream() = default;
+ ~MemoryOutStream() = default;
+
+ virtual void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset) override;
+ virtual void Flush() override;
+ inline const uint8_t* Data() const { return m_Buffer.data(); }
+ inline uint64_t Size() const { return m_Buffer.size(); }
+
+private:
+ RwLock m_Lock;
+ std::vector<uint8_t> m_Buffer;
+};
+
+inline MemoryView
+MakeMemoryView(const MemoryOutStream& Stream)
+{
+ return MemoryView(Stream.Data(), Stream.Size());
+}
+
+/**
+ * Stream which reads from a memory buffer
+ */
+class MemoryInStream : public InStream
+{
+public:
+ MemoryInStream(const void* Buffer, size_t Size);
+ MemoryInStream(MemoryView View) : MemoryInStream(View.GetData(), View.GetSize()) {}
+ ~MemoryInStream() = default;
+
+ virtual void Read(void* DataPtr, size_t ByteCount, uint64_t ReadOffset) override;
+ virtual uint64_t Size() const override { return m_Buffer.size(); }
+ inline const uint8_t* Data() const { return m_Buffer.data(); }
+
+private:
+ RwLock m_Lock;
+ std::vector<uint8_t> m_Buffer;
+};
+
+/**
+ * Binary stream writer
+ */
+
+class BinaryWriter
+{
+public:
+ inline BinaryWriter(OutStream& Stream) : m_Stream(&Stream) {}
+ ~BinaryWriter() = default;
+
+ inline void Write(const void* DataPtr, size_t ByteCount)
+ {
+ m_Stream->Write(DataPtr, ByteCount, m_Offset);
+ m_Offset += ByteCount;
+ }
+
+ uint64_t CurrentOffset() const { return m_Offset; }
+
+private:
+ RefPtr<OutStream> m_Stream;
+ uint64_t m_Offset = 0;
+};
+
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, bool Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, int8_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, int16_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, int32_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, int64_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, uint8_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, uint16_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, uint32_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, uint64_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+
+/**
+ * Binary stream reader
+ */
+
+class BinaryReader
+{
+public:
+ inline BinaryReader(InStream& Stream) : m_Stream(&Stream) {}
+ ~BinaryReader() = default;
+
+ inline void Read(void* DataPtr, size_t ByteCount)
+ {
+ m_Stream->Read(DataPtr, ByteCount, m_Offset);
+ m_Offset += ByteCount;
+ }
+
+ void Seek(uint64_t Offset)
+ {
+ ZEN_ASSERT(Offset <= m_Stream->Size());
+ m_Offset = Offset;
+ }
+
+ void Skip(uint64_t SkipOffset)
+ {
+ ZEN_ASSERT((m_Offset + SkipOffset) <= m_Stream->Size());
+ m_Offset += SkipOffset;
+ }
+
+ inline uint64_t CurrentOffset() const { return m_Offset; }
+ inline uint64_t AvailableBytes() const { return m_Stream->Size() - m_Offset; }
+
+private:
+ RefPtr<InStream> m_Stream;
+ uint64_t m_Offset = 0;
+};
+
+inline BinaryReader&
+operator>>(BinaryReader& Reader, bool& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, int8_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, int16_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, int32_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, int64_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, uint8_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, uint16_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, uint32_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, uint64_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+
+/**
+ * Text stream writer
+ */
+
+class TextWriter
+{
+public:
+ ZENCORE_API TextWriter(OutStream& Stream);
+ ZENCORE_API ~TextWriter();
+
+ ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount);
+ ZENCORE_API void Writef(const char* FormatString, ...);
+
+ inline uint64_t CurrentOffset() const { return m_CurrentOffset; }
+
+private:
+ RefPtr<OutStream> m_Stream;
+ uint64_t m_CurrentOffset = 0;
+};
+
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const char* Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const std::string_view& Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, bool Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int8_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int16_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int32_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int64_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint8_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint16_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint32_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint64_t Value);
+
+class IndentTextWriter : public TextWriter
+{
+public:
+ ZENCORE_API IndentTextWriter(OutStream& stream);
+ ZENCORE_API ~IndentTextWriter();
+
+ ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount) override;
+
+ inline void Indent(int Amount) { m_IndentAmount += Amount; }
+
+ struct Scope
+ {
+ Scope(IndentTextWriter& Outer, int IndentAmount = 2) : m_Outer(Outer), m_IndentAmount(IndentAmount)
+ {
+ m_Outer.Indent(IndentAmount);
+ }
+
+ ~Scope() { m_Outer.Indent(-m_IndentAmount); }
+
+ private:
+ IndentTextWriter& m_Outer;
+ int m_IndentAmount;
+ };
+
+private:
+ int m_IndentAmount = 0;
+ int m_LineCursor = 0;
+ char m_LineBuffer[2048];
+};
+
+void stream_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/streamutil.h b/zencore/include/zencore/streamutil.h
new file mode 100644
index 000000000..190cd18eb
--- /dev/null
+++ b/zencore/include/zencore/streamutil.h
@@ -0,0 +1,118 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <fmt/format.h>
+#include <zencore/string.h>
+#include <string>
+#include <string_view>
+
+#include "blake3.h"
+#include "iohash.h"
+#include "sha1.h"
+#include "stream.h"
+
+namespace zen {
+
+ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::string_view& value);
+ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::string& value);
+
+ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::wstring_view& value);
+ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::wstring& value);
+ZENCORE_API TextWriter& operator<<(TextWriter& writer, const std::wstring_view& value);
+
+inline BinaryWriter&
+operator<<(BinaryWriter& writer, const SHA1& value)
+{
+ writer.Write(value.Hash, sizeof value.Hash);
+ return writer;
+}
+inline BinaryReader&
+operator>>(BinaryReader& reader, SHA1& value)
+{
+ reader.Read(value.Hash, sizeof value.Hash);
+ return reader;
+}
+ZENCORE_API TextWriter& operator<<(TextWriter& writer, const zen::SHA1& value);
+
+inline BinaryWriter&
+operator<<(BinaryWriter& writer, const BLAKE3& value)
+{
+ writer.Write(value.Hash, sizeof value.Hash);
+ return writer;
+}
+inline BinaryReader&
+operator>>(BinaryReader& reader, BLAKE3& value)
+{
+ reader.Read(value.Hash, sizeof value.Hash);
+ return reader;
+}
+ZENCORE_API TextWriter& operator<<(TextWriter& writer, const BLAKE3& value);
+
+inline BinaryWriter&
+operator<<(BinaryWriter& writer, const IoHash& value)
+{
+ writer.Write(value.Hash, sizeof value.Hash);
+ return writer;
+}
+inline BinaryReader&
+operator>>(BinaryReader& reader, IoHash& value)
+{
+ reader.Read(value.Hash, sizeof value.Hash);
+ return reader;
+}
+ZENCORE_API TextWriter& operator<<(TextWriter& writer, const IoHash& value);
+
+} // namespace zen
+
+//////////////////////////////////////////////////////////////////////////
+
+template<>
+struct fmt::formatter<zen::IoHash>
+{
+ constexpr auto parse(format_parse_context& ctx)
+ {
+ // Parse the presentation format and store it in the formatter:
+ auto it = ctx.begin(), end = ctx.end();
+
+ // Check if reached the end of the range:
+ if (it != end && *it != '}')
+ throw format_error("invalid format");
+
+ // Return an iterator past the end of the parsed range:
+ return it;
+ }
+
+ template<typename FormatContext>
+ auto format(const zen::IoHash& h, FormatContext& ctx)
+ {
+ zen::ExtendableStringBuilder<48> String;
+ h.ToHexString(String);
+ return format_to(ctx.out(), std::string_view(String));
+ }
+};
+
+template<>
+struct fmt::formatter<zen::BLAKE3>
+{
+ constexpr auto parse(format_parse_context& ctx)
+ {
+ // Parse the presentation format and store it in the formatter:
+ auto it = ctx.begin(), end = ctx.end();
+
+ // Check if reached the end of the range:
+ if (it != end && *it != '}')
+ throw format_error("invalid format");
+
+ // Return an iterator past the end of the parsed range:
+ return it;
+ }
+
+ template<typename FormatContext>
+ auto format(const zen::BLAKE3& h, FormatContext& ctx)
+ {
+ zen::ExtendableStringBuilder<80> String;
+ h.ToHexString(String);
+ return format_to(ctx.out(), std::string_view(String));
+ }
+};
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
new file mode 100644
index 000000000..d7727ca08
--- /dev/null
+++ b/zencore/include/zencore/string.h
@@ -0,0 +1,595 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "intmath.h"
+#include "zencore.h"
+
+#include <stdint.h>
+#include <string.h>
+#include <charconv>
+#include <codecvt>
+#include <concepts>
+#include <optional>
+#include <span>
+#include <string_view>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+inline bool
+StringEquals(const char8_t* s1, const char* s2)
+{
+ return strcmp(reinterpret_cast<const char*>(s1), s2) == 0;
+}
+
+inline bool
+StringEquals(const char* s1, const char* s2)
+{
+ return strcmp(s1, s2) == 0;
+}
+
+inline size_t
+StringLength(const char* str)
+{
+ return strlen(str);
+}
+
+inline bool
+StringEquals(const wchar_t* s1, const wchar_t* s2)
+{
+ return wcscmp(s1, s2) == 0;
+}
+
+inline size_t
+StringLength(const wchar_t* str)
+{
+ return wcslen(str);
+}
+
+//////////////////////////////////////////////////////////////////////////
+// File name helpers
+//
+
+ZENCORE_API const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr);
+
+//////////////////////////////////////////////////////////////////////////
+// Text formatting of numbers
+//
+
+ZENCORE_API bool ToString(std::span<char> Buffer, uint64_t Num);
+ZENCORE_API bool ToString(std::span<char> Buffer, int64_t Num);
+
+struct TextNumBase
+{
+ inline const char* c_str() const { return m_Buffer; }
+ inline operator std::string_view() const { return std::string_view(m_Buffer); }
+
+protected:
+ char m_Buffer[24];
+};
+
+struct IntNum : public TextNumBase
+{
+ inline IntNum(std::unsigned_integral auto Number) { ToString(m_Buffer, uint64_t(Number)); }
+ inline IntNum(std::signed_integral auto Number) { ToString(m_Buffer, int64_t(Number)); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Quick-and-dirty string builder. Good enough for me, but contains traps
+// and not-quite-ideal behaviour especially when mixing character types etc
+//
+
+template<typename C>
+class StringBuilderImpl
+{
+public:
+ StringBuilderImpl() = default;
+ ZENCORE_API ~StringBuilderImpl();
+
+ StringBuilderImpl(const StringBuilderImpl&) = delete;
+ StringBuilderImpl(const StringBuilderImpl&&) = delete;
+ const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete;
+ const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete;
+
+ StringBuilderImpl& Append(C OneChar)
+ {
+ EnsureCapacity(1);
+
+ *m_CurPos++ = OneChar;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const std::string_view& String)
+ {
+ const size_t len = String.size();
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = String[i];
+
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const std::u8string_view& String)
+ {
+ const size_t len = String.size();
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = String[i];
+
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const char* NulTerminatedString)
+ {
+ size_t StringLen = StringLength(NulTerminatedString);
+
+ return AppendAscii({NulTerminatedString, StringLen});
+ }
+
+ inline StringBuilderImpl& Append(const char8_t* NulTerminatedString)
+ {
+ // This is super hacky and not fully functional - needs better
+ // solution
+ if constexpr (sizeof(C) == 1)
+ {
+ size_t len = StringLength((const char*)NulTerminatedString);
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = C(NulTerminatedString[i]);
+
+ m_CurPos += len;
+ }
+ else
+ {
+ ZEN_NOT_IMPLEMENTED();
+ }
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAsciiRange(const char* BeginString, const char* EndString)
+ {
+ EnsureCapacity(EndString - BeginString);
+
+ while (BeginString != EndString)
+ *m_CurPos++ = *BeginString++;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const C* NulTerminatedString)
+ {
+ size_t Len = StringLength(NulTerminatedString);
+
+ EnsureCapacity(Len);
+ memcpy(m_CurPos, NulTerminatedString, Len * sizeof(C));
+ m_CurPos += Len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const C* NulTerminatedString, size_t MaxChars)
+ {
+ size_t len = Min(MaxChars, StringLength(NulTerminatedString));
+
+ EnsureCapacity(len);
+ memcpy(m_CurPos, NulTerminatedString, len * sizeof(C));
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendRange(const C* BeginString, const C* EndString)
+ {
+ size_t Len = EndString - BeginString;
+
+ EnsureCapacity(Len);
+ memcpy(m_CurPos, BeginString, Len * sizeof(C));
+ m_CurPos += Len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const std::basic_string_view<C>& String)
+ {
+ return AppendRange(String.data(), String.data() + String.size());
+ }
+
+ inline const C* c_str() const
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline C* Data()
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline const C* Data() const
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline size_t Size() const { return m_CurPos - m_Base; }
+ inline bool IsDynamic() const { return m_IsDynamic; }
+ inline void Reset() { m_CurPos = m_Base; }
+
+ inline StringBuilderImpl& operator<<(uint64_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int64_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint32_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int32_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint16_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int16_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint8_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int8_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+
+ inline StringBuilderImpl& operator<<(const char* str) { return AppendAscii(str); }
+ inline StringBuilderImpl& operator<<(const std::string_view str) { return AppendAscii(str); }
+ inline StringBuilderImpl& operator<<(const std::u8string_view str) { return AppendAscii(str); }
+ inline StringBuilderImpl& operator<<(bool v)
+ {
+ using namespace std::literals;
+ if (v)
+ {
+ return AppendAscii("true"sv);
+ }
+ return AppendAscii("false"sv);
+ }
+
+protected:
+ inline void Init(C* Base, size_t Capacity)
+ {
+ m_Base = m_CurPos = Base;
+ m_End = Base + Capacity;
+ }
+
+ inline void EnsureNulTerminated() const { *m_CurPos = '\0'; }
+
+ inline void EnsureCapacity(size_t ExtraRequired)
+ {
+ // precondition: we know the current buffer has enough capacity
+ // for the existing string including NUL terminator
+
+ if ((m_CurPos + ExtraRequired) < m_End)
+ return;
+
+ Extend(ExtraRequired);
+ }
+
+ ZENCORE_API void Extend(size_t ExtraCapacity);
+ ZENCORE_API void* AllocBuffer(size_t ByteCount);
+ ZENCORE_API void FreeBuffer(void* Buffer, size_t ByteCount);
+
+ ZENCORE_API [[noreturn]] void Fail(const char* FailReason); // note: throws exception
+
+ C* m_Base;
+ C* m_CurPos;
+ C* m_End;
+ bool m_IsDynamic = false;
+ bool m_IsExtendable = false;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+extern template StringBuilderImpl<char>;
+
+class StringBuilderBase : public StringBuilderImpl<char>
+{
+public:
+ inline StringBuilderBase(char* bufferPointer, size_t bufferCapacity) { Init(bufferPointer, bufferCapacity); }
+ inline ~StringBuilderBase() = default;
+
+ // Note that we don't need a terminator for the string_view so we avoid calling data() here
+ inline operator std::string_view() const { return std::string_view(m_Base, m_CurPos - m_Base); }
+ inline std::string_view ToView() const { return std::string_view(m_Base, m_CurPos - m_Base); }
+ inline std::string ToString() const { return std::string{Data(), Size()}; }
+
+ inline void AppendCodepoint(uint32_t cp)
+ {
+ if (cp < 0x80) // one octet
+ {
+ Append(static_cast<char8_t>(cp));
+ }
+ else if (cp < 0x800)
+ {
+ EnsureCapacity(2); // two octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 6) | 0xc0);
+ m_CurPos[1] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 2;
+ }
+ else if (cp < 0x10000)
+ {
+ EnsureCapacity(3); // three octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 12) | 0xe0);
+ m_CurPos[1] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80);
+ m_CurPos[2] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 3;
+ }
+ else
+ {
+ EnsureCapacity(4); // four octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 18) | 0xf0);
+ m_CurPos[1] = static_cast<char8_t>(((cp >> 12) & 0x3f) | 0x80);
+ m_CurPos[2] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80);
+ m_CurPos[3] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 4;
+ }
+ }
+};
+
+template<size_t N>
+class StringBuilder : public StringBuilderBase
+{
+public:
+ inline StringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) {}
+ inline ~StringBuilder() = default;
+
+private:
+ char m_StringBuffer[N];
+};
+
+template<size_t N>
+class ExtendableStringBuilder : public StringBuilderBase
+{
+public:
+ inline ExtendableStringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) { m_IsExtendable = true; }
+ inline ~ExtendableStringBuilder() = default;
+
+private:
+ char m_StringBuffer[N];
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+extern template StringBuilderImpl<wchar_t>;
+
+class WideStringBuilderBase : public StringBuilderImpl<wchar_t>
+{
+public:
+ inline WideStringBuilderBase(wchar_t* BufferPointer, size_t BufferCapacity) { Init(BufferPointer, BufferCapacity); }
+ inline ~WideStringBuilderBase() = default;
+
+ inline operator std::wstring_view() const { return std::wstring_view{Data(), Size()}; }
+ inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; }
+ inline std::wstring toString() const { return std::wstring{Data(), Size()}; }
+
+ inline StringBuilderImpl& operator<<(const std::u16string_view str) { return Append((const wchar_t*)str.data(), str.size()); }
+ inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); }
+ using StringBuilderImpl:: operator<<;
+};
+
+template<size_t N>
+class WideStringBuilder : public WideStringBuilderBase
+{
+public:
+ inline WideStringBuilder() : WideStringBuilderBase(m_Buffer, N) {}
+ ~WideStringBuilder() = default;
+
+private:
+ wchar_t m_Buffer[N];
+};
+
+template<size_t N>
+class ExtendableWideStringBuilder : public WideStringBuilderBase
+{
+public:
+ inline ExtendableWideStringBuilder() : WideStringBuilderBase(m_Buffer, N) { m_IsExtendable = true; }
+ ~ExtendableWideStringBuilder() = default;
+
+private:
+ wchar_t m_Buffer[N];
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out);
+void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out);
+void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out);
+std::wstring Utf8ToWide(const std::string_view& wstr);
+
+void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out);
+std::string WideToUtf8(const wchar_t* wstr);
+void WideToUtf8(const std::u16string_view& wstr, StringBuilderBase& out);
+void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out);
+std::string WideToUtf8(const std::wstring_view Wstr);
+
+/// <summary>
+/// Parse hex string into a byte buffer
+/// </summary>
+/// <param name="string">Input string</param>
+/// <param name="characterCount">Number of characters in string</param>
+/// <param name="outPtr">Pointer to output buffer</param>
+/// <returns>true if the input consisted of all valid hexadecimal characters</returns>
+
+inline bool
+ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr)
+{
+ ZEN_ASSERT((CharacterCount & 1) == 0);
+
+ auto char2nibble = [](char c) {
+ uint8_t c8 = uint8_t(c - '0');
+
+ if (c8 < 10)
+ return c8;
+
+ c8 -= 'A' - '0' - 10;
+
+ if (c8 < 16)
+ return c8;
+
+ c8 -= 'a' - 'A';
+
+ if (c8 < 16)
+ return c8;
+
+ return uint8_t(0xff);
+ };
+
+ uint8_t allBits = 0;
+
+ while (CharacterCount)
+ {
+ uint8_t n0 = char2nibble(InputString[0]);
+ uint8_t n1 = char2nibble(InputString[1]);
+
+ allBits |= n0 | n1;
+
+ *OutPtr = (n0 << 4) | n1;
+
+ OutPtr += 1;
+ InputString += 2;
+ CharacterCount -= 2;
+ }
+
+ return (allBits & 0x80) == 0;
+}
+
+inline void
+ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString)
+{
+ const char hexchars[] = "0123456789abcdef";
+
+ while (ByteCount--)
+ {
+ uint8_t byte = *InputData++;
+
+ *OutString++ = hexchars[byte >> 4];
+ *OutString++ = hexchars[byte & 15];
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+// Format numbers for humans
+//
+
+ZENCORE_API size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer);
+ZENCORE_API size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer);
+ZENCORE_API size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer);
+ZENCORE_API size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer);
+ZENCORE_API size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer);
+
+struct NiceBase
+{
+ inline const char* c_str() const { return m_Buffer; }
+ inline operator std::string_view() const { return std::string_view(m_Buffer); }
+
+protected:
+ char m_Buffer[16];
+};
+
+struct NiceNum : public NiceBase
+{
+ inline NiceNum(uint64_t Num) { NiceNumToBuffer(Num, m_Buffer); }
+};
+
+struct NiceBytes : public NiceBase
+{
+ inline NiceBytes(uint64_t Num) { NiceBytesToBuffer(Num, m_Buffer); }
+};
+
+struct NiceByteRate : public NiceBase
+{
+ inline NiceByteRate(uint64_t Bytes, uint64_t TimeMilliseconds) { NiceByteRateToBuffer(Bytes, TimeMilliseconds, m_Buffer); }
+};
+
+struct NiceLatencyNs : public NiceBase
+{
+ inline NiceLatencyNs(uint64_t Milliseconds) { NiceLatencyNsToBuffer(Milliseconds, m_Buffer); }
+};
+
+struct NiceTimeSpanMs : public NiceBase
+{
+ inline NiceTimeSpanMs(uint64_t Milliseconds) { NiceTimeSpanMsToBuffer(Milliseconds, m_Buffer); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+inline std::string
+NiceRate(uint64_t Num, uint32_t DurationMilliseconds, const char* Unit = "B")
+{
+ char Buffer[32];
+
+ if (DurationMilliseconds)
+ {
+ NiceNumToBuffer(Num * 1000 / DurationMilliseconds, Buffer);
+ }
+ else
+ {
+ strcpy_s(Buffer, "0");
+ }
+
+ strcat_s(Buffer, Unit);
+ strcat_s(Buffer, "/s");
+
+ return Buffer;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template<std::integral T>
+std::optional<T>
+ParseInt(const std::string_view& Input)
+{
+ T Out;
+ const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out);
+ if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range)
+ {
+ return std::nullopt;
+ }
+ return Out;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void string_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h
new file mode 100644
index 000000000..48afad33f
--- /dev/null
+++ b/zencore/include/zencore/thread.h
@@ -0,0 +1,118 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+/**
+ * Reader-writer lock
+ *
+ * - A single thread may hold an exclusive lock at any given moment
+ *
+ * - Multiple threads may hold shared locks, but only if no thread has
+ * acquired an exclusive lock
+ */
+class RwLock
+{
+public:
+ ZENCORE_API void AcquireShared();
+ ZENCORE_API void ReleaseShared();
+
+ ZENCORE_API void AcquireExclusive();
+ ZENCORE_API void ReleaseExclusive();
+
+ struct SharedLockScope
+ {
+ SharedLockScope(RwLock& lock) : m_Lock(lock) { m_Lock.AcquireShared(); }
+ ~SharedLockScope() { m_Lock.ReleaseShared(); }
+
+ private:
+ RwLock& m_Lock;
+ };
+
+ struct ExclusiveLockScope
+ {
+ ExclusiveLockScope(RwLock& lock) : m_Lock(lock) { m_Lock.AcquireExclusive(); }
+ ~ExclusiveLockScope() { m_Lock.ReleaseExclusive(); }
+
+ private:
+ RwLock& m_Lock;
+ };
+
+private:
+ void* m_Srw = nullptr;
+};
+
+/** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore')
+ */
+class Event
+{
+public:
+ ZENCORE_API Event();
+ ZENCORE_API ~Event();
+
+ Event(Event&& Rhs) : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; }
+
+ Event(const Event& Rhs) = delete;
+ Event& operator=(const Event& Rhs) = delete;
+
+ inline Event& operator=(Event&& Rhs)
+ {
+ m_EventHandle = Rhs.m_EventHandle;
+ Rhs.m_EventHandle = nullptr;
+ return *this;
+ }
+
+ ZENCORE_API void Set();
+ ZENCORE_API void Reset();
+ ZENCORE_API bool Wait(int TimeoutMs = -1);
+
+protected:
+ explicit Event(void* EventHandle) : m_EventHandle(EventHandle) {}
+
+ void* m_EventHandle = nullptr;
+};
+
+/** Basic abstraction of an IPC mechanism (aka 'binary semaphore')
+ */
+class NamedEvent : public Event
+{
+public:
+ ZENCORE_API explicit NamedEvent(std::string_view EventName);
+ ZENCORE_API explicit NamedEvent(std::u8string_view EventName);
+};
+
+/** Basic process abstraction
+ */
+class Process
+{
+public:
+ ZENCORE_API Process();
+
+ Process(const Process&) = delete;
+ Process& operator=(const Process&) = delete;
+
+ ZENCORE_API ~Process();
+
+ ZENCORE_API void Initialize(int Pid);
+ ZENCORE_API void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle
+ ZENCORE_API bool IsRunning() const;
+ ZENCORE_API bool IsValid() const;
+ ZENCORE_API bool Wait(int TimeoutMs = -1);
+ ZENCORE_API void Terminate(int ExitCode);
+ inline int Pid() const { return m_Pid; }
+
+private:
+ void* m_ProcessHandle = nullptr;
+ int m_Pid = 0;
+};
+
+ZENCORE_API bool IsProcessRunning(int pid);
+
+ZENCORE_API void Sleep(int ms);
+
+void thread_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/timer.h b/zencore/include/zencore/timer.h
new file mode 100644
index 000000000..c9122eb44
--- /dev/null
+++ b/zencore/include/zencore/timer.h
@@ -0,0 +1,41 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <intrin.h>
+#include <stdint.h>
+#include "zencore.h"
+
+namespace zen {
+
+// High frequency timers
+
+ZENCORE_API uint64_t GetHifreqTimerValue();
+ZENCORE_API uint64_t GetHifreqTimerFrequency();
+ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init
+
+class Stopwatch
+{
+public:
+ Stopwatch() : m_StartValue(GetHifreqTimerValue()) {}
+
+ inline uint64_t getElapsedTimeMs() { return (GetHifreqTimerValue() - m_StartValue) * 1000 / GetHifreqTimerFrequency(); }
+
+ inline void reset() { m_StartValue = GetHifreqTimerValue(); }
+
+private:
+ uint64_t m_StartValue;
+};
+
+// CPU timers
+
+inline uint64_t
+GetCpuTimerValue()
+{
+ unsigned int foo;
+ return __rdtscp(&foo);
+}
+
+void timer_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/trace.h b/zencore/include/zencore/trace.h
new file mode 100644
index 000000000..191ce4a3a
--- /dev/null
+++ b/zencore/include/zencore/trace.h
@@ -0,0 +1,91 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <inttypes.h>
+#include "zencore.h"
+
+#pragma section("trace_events", read)
+#define U_TRACE_DECL __declspec(allocate("trace_events"))
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace zen {
+
+struct TraceSite
+{
+ const char* sourceFile;
+ const uint32_t sourceLine;
+ const uint32_t flags;
+};
+
+struct TraceEvent
+{
+ const TraceSite* site;
+ ThreadId_t threadId;
+ const char* message;
+};
+
+enum TraceFlags
+{
+ kTrace_Debug = 1 << 0,
+ kTrace_Info = 1 << 1,
+ kTrace_Warn = 1 << 2,
+ kTrace_Error = 1 << 3,
+ kTrace_Fatal = 1 << 4,
+
+ kTrace_Trace = 1 << 7,
+};
+
+class Tracer
+{
+public:
+ void Log(const TraceEvent& e);
+
+ __forceinline uint32_t Accept(const TraceSite& e) const { return (m_acceptFlags & e.flags); }
+
+private:
+ uint32_t m_acceptFlags = ~0u;
+};
+
+ZENCORE_API extern Tracer g_globalTracer;
+
+/** Trace event handler
+ */
+class TraceHandler
+{
+public:
+ virtual void Trace(const TraceEvent& e) = 0;
+
+private:
+};
+
+ZENCORE_API static void TraceBroadcast(const TraceEvent& e);
+
+void trace_forcelink(); // internal
+
+} // namespace zen
+
+__forceinline zen::Tracer&
+CurrentTracer()
+{
+ return zen::g_globalTracer;
+}
+
+#define U_LOG_GENERIC(msg, flags) \
+ do \
+ { \
+ zen::Tracer& t = CurrentTracer(); \
+ static U_TRACE_DECL constexpr zen::TraceSite traceSite{__FILE__, __LINE__, flags}; \
+ const zen::TraceEvent traceEvent = {&traceSite, 0u, msg}; \
+ if (t.Accept(traceSite)) \
+ t.Log(traceEvent); \
+ } while (false)
+
+//////////////////////////////////////////////////////////////////////////
+
+#define U_LOG_DEBUG(msg) U_LOG_GENERIC(msg, zen::kTrace_Debug)
+#define U_LOG_INFO(msg) U_LOG_GENERIC(msg, zen::kTrace_Info)
+#define U_LOG_WARN(msg) U_LOG_GENERIC(msg, zen::kTrace_Warn)
+#define U_LOG_ERROR(msg) U_LOG_GENERIC(msg, zen::kTrace_Error)
+#define U_LOG_FATAL(msg) U_LOG_GENERIC(msg, zen::kTrace_Fatal)
diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h
new file mode 100644
index 000000000..a793b160a
--- /dev/null
+++ b/zencore/include/zencore/uid.h
@@ -0,0 +1,78 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+#include <compare>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/** Object identifier
+
+ Can be used as a GUID essentially, but is more compact (12 bytes) and as such
+ is more susceptible to collisions than a 16-byte GUID but also I don't expect
+ the population to be large so in practice the risk should be minimal due to
+ how the identifiers work.
+
+ Similar in spirit to MongoDB ObjectId
+
+ When serialized, object identifiers generated in a given session in sequence
+ will sort in chronological order since the timestamp is in the MSB in big
+ endian format. This makes it suitable as a database key since most indexing
+ structures work better when keys are inserted in lexicographically
+ increasing order.
+
+ The current layout is basically:
+
+ |----------------|----------------|----------------|
+ | timestamp | serial # | run id |
+ |----------------|----------------|----------------|
+ MSB LSB
+
+ - Timestamp is a unsigned 32-bit value (seconds since Jan 1 1970)
+ - Serial # is another unsigned 32-bit value which is assigned a (strong)
+ random number at initialization time which is incremented when a new Oid
+ is generated
+ - The run id is generated from a strong random number generator
+ at initialization time and stays fixed for the duration of the program
+
+ Timestamp and serial are stored in memory in such a way that they can be
+ ordered lexicographically. I.e they are in big-endian byte order.
+
+ */
+
+struct Oid
+{
+ static const int StringLength = 24;
+ typedef char String_t[StringLength + 1];
+
+ static void Initialize();
+ [[nodiscard]] static Oid NewOid();
+
+ const Oid& Generate();
+ [[nodiscard]] static Oid FromHexString(const std::string_view String);
+ StringBuilderBase& ToString(StringBuilderBase& OutString) const;
+
+ auto operator<=>(const Oid& rhs) const = default;
+
+ static const Oid Zero; // Min (can be used to signify a "null" value, or for open range queries)
+ static const Oid Max; // Max (can be used for open range queries)
+
+ struct Hasher
+ {
+ size_t operator()(const Oid& id) const
+ {
+ const size_t seed = id.OidBits[0];
+ return (seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1]) | (uint64_t(id.OidBits[2]) << 32);
+ }
+ };
+
+ // You should not assume anything about these words
+ uint32_t OidBits[3];
+};
+
+extern void uid_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/varint.h b/zencore/include/zencore/varint.h
new file mode 100644
index 000000000..0c40dd66b
--- /dev/null
+++ b/zencore/include/zencore/varint.h
@@ -0,0 +1,255 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "intmath.h"
+
+namespace zen {
+
+// Variable-Length Integer Encoding
+//
+// ZigZag encoding is used to convert signed integers into unsigned integers in a way that allows
+// integers with a small magnitude to have a smaller encoded representation.
+//
+// An unsigned integer is encoded into 1-9 bytes based on its magnitude. The first byte indicates
+// how many additional bytes are used by the number of leading 1-bits that it has. The additional
+// bytes are stored in big endian order, and the most significant bits of the value are stored in
+// the remaining bits in the first byte. The encoding of the first byte allows the reader to skip
+// over the encoded integer without consuming its bytes individually.
+//
+// Encoded unsigned integers sort the same in a byte-wise comparison as when their decoded values
+// are compared. The same property does not hold for signed integers due to ZigZag encoding.
+//
+// 32-bit inputs encode to 1-5 bytes.
+// 64-bit inputs encode to 1-9 bytes.
+//
+// 0x0000'0000'0000'0000 - 0x0000'0000'0000'007f : 0b0_______ 1 byte
+// 0x0000'0000'0000'0080 - 0x0000'0000'0000'3fff : 0b10______ 2 bytes
+// 0x0000'0000'0000'4000 - 0x0000'0000'001f'ffff : 0b110_____ 3 bytes
+// 0x0000'0000'0020'0000 - 0x0000'0000'0fff'ffff : 0b1110____ 4 bytes
+// 0x0000'0000'1000'0000 - 0x0000'0007'ffff'ffff : 0b11110___ 5 bytes
+// 0x0000'0008'0000'0000 - 0x0000'03ff'ffff'ffff : 0b111110__ 6 bytes
+// 0x0000'0400'0000'0000 - 0x0001'ffff'ffff'ffff : 0b1111110_ 7 bytes
+// 0x0002'0000'0000'0000 - 0x00ff'ffff'ffff'ffff : 0b11111110 8 bytes
+// 0x0100'0000'0000'0000 - 0xffff'ffff'ffff'ffff : 0b11111111 9 bytes
+//
+// Encoding Examples
+// -42 => ZigZag => 0x53 => 0x53
+// 42 => ZigZag => 0x54 => 0x54
+// 0x1 => 0x01
+// 0x12 => 0x12
+// 0x123 => 0x81 0x23
+// 0x1234 => 0x92 0x34
+// 0x12345 => 0xc1 0x23 0x45
+// 0x123456 => 0xd2 0x34 0x56
+// 0x1234567 => 0xe1 0x23 0x45 0x67
+// 0x12345678 => 0xf0 0x12 0x34 0x56 0x78
+// 0x123456789 => 0xf1 0x23 0x45 0x67 0x89
+// 0x123456789a => 0xf8 0x12 0x34 0x56 0x78 0x9a
+// 0x123456789ab => 0xfb 0x23 0x45 0x67 0x89 0xab
+// 0x123456789abc => 0xfc 0x12 0x34 0x56 0x78 0x9a 0xbc
+// 0x123456789abcd => 0xfd 0x23 0x45 0x67 0x89 0xab 0xcd
+// 0x123456789abcde => 0xfe 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde
+// 0x123456789abcdef => 0xff 0x01 0x23 0x45 0x67 0x89 0xab 0xcd 0xef
+// 0x123456789abcdef0 => 0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0
+
+/**
+ * Measure the length in bytes (1-9) of an encoded variable-length integer.
+ *
+ * @param InData A variable-length encoding of an (signed or unsigned) integer.
+ * @return The number of bytes used to encode the integer, in the range 1-9.
+ */
+inline uint32_t
+MeasureVarUInt(const void* InData)
+{
+ return CountLeadingZeros(uint8_t(~*static_cast<const uint8_t*>(InData))) - 23;
+}
+
+/** Measure the length in bytes (1-9) of an encoded variable-length integer. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(const void* InData)
+{
+ return MeasureVarUInt(InData);
+}
+
+/** Measure the number of bytes (1-5) required to encode the 32-bit input. */
+inline uint32_t
+MeasureVarUInt(uint32_t InValue)
+{
+ return uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1);
+}
+
+/** Measure the number of bytes (1-9) required to encode the 64-bit input. */
+inline uint32_t
+MeasureVarUInt(uint64_t InValue)
+{
+ return uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9));
+}
+
+/** Measure the number of bytes (1-5) required to encode the 32-bit input. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(int32_t InValue)
+{
+ return MeasureVarUInt(uint32_t((InValue >> 31) ^ (InValue << 1)));
+}
+
+/** Measure the number of bytes (1-9) required to encode the 64-bit input. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(int64_t InValue)
+{
+ return MeasureVarUInt(uint64_t((InValue >> 63) ^ (InValue << 1)));
+}
+
+/**
+ * Read a variable-length unsigned integer.
+ *
+ * @param InData A variable-length encoding of an unsigned integer.
+ * @param OutByteCount The number of bytes consumed from the input.
+ * @return An unsigned integer.
+ */
+inline uint64_t
+ReadVarUInt(const void* InData, uint32_t& OutByteCount)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InData);
+ OutByteCount = ByteCount;
+
+ const uint8_t* InBytes = static_cast<const uint8_t*>(InData);
+ uint64_t Value = *InBytes++ & uint8_t(0xff >> ByteCount);
+ switch (ByteCount - 1)
+ {
+ case 8:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 7:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 6:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 5:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 4:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 3:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 2:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 1:
+ Value <<= 8;
+ Value |= *InBytes++;
+ default:
+ return Value;
+ }
+}
+
+/**
+ * Read a variable-length signed integer.
+ *
+ * @param InData A variable-length encoding of a signed integer.
+ * @param OutByteCount The number of bytes consumed from the input.
+ * @return A signed integer.
+ */
+inline int64_t
+ReadVarInt(const void* InData, uint32_t& OutByteCount)
+{
+ const uint64_t Value = ReadVarUInt(InData, OutByteCount);
+ return -int64_t(Value & 1) ^ int64_t(Value >> 1);
+}
+
+/**
+ * Write a variable-length unsigned integer.
+ *
+ * @param InValue An unsigned integer to encode.
+ * @param OutData A buffer of at least 5 bytes to write the output to.
+ * @return The number of bytes used in the output.
+ */
+inline uint32_t
+WriteVarUInt(uint32_t InValue, void* OutData)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InValue);
+ uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1;
+ switch (ByteCount - 1)
+ {
+ case 4:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 3:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 2:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 1:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ default:
+ break;
+ }
+ *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue);
+ return ByteCount;
+}
+
+/**
+ * Write a variable-length unsigned integer.
+ *
+ * @param InValue An unsigned integer to encode.
+ * @param OutData A buffer of at least 9 bytes to write the output to.
+ * @return The number of bytes used in the output.
+ */
+inline uint32_t
+WriteVarUInt(uint64_t InValue, void* OutData)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InValue);
+ uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1;
+ switch (ByteCount - 1)
+ {
+ case 8:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 7:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 6:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 5:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 4:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 3:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 2:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 1:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ default:
+ break;
+ }
+ *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue);
+ return ByteCount;
+}
+
+/** Write a variable-length signed integer. \see \ref WriteVarUInt */
+inline uint32_t
+WriteVarInt(int32_t InValue, void* OutData)
+{
+ const uint32_t Value = uint32_t((InValue >> 31) ^ (InValue << 1));
+ return WriteVarUInt(Value, OutData);
+}
+
+/** Write a variable-length signed integer. \see \ref WriteVarUInt */
+inline uint32_t
+WriteVarInt(int64_t InValue, void* OutData)
+{
+ const uint64_t Value = uint64_t((InValue >> 63) ^ (InValue << 1));
+ return WriteVarUInt(Value, OutData);
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/windows.h b/zencore/include/zencore/windows.h
new file mode 100644
index 000000000..8888bf757
--- /dev/null
+++ b/zencore/include/zencore/windows.h
@@ -0,0 +1,10 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive-
+#ifndef NOMINMAX
+# define NOMINMAX // We don't want your min/max macros
+#endif
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
diff --git a/zencore/include/zencore/xxhash.h b/zencore/include/zencore/xxhash.h
new file mode 100644
index 000000000..5407755df
--- /dev/null
+++ b/zencore/include/zencore/xxhash.h
@@ -0,0 +1,87 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/memory.h>
+
+#include <xxh3.h>
+#include <string_view>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/**
+ * XXH3 hash
+ */
+struct XXH3_128
+{
+ uint8_t Hash[16];
+
+ static XXH3_128 MakeFrom(const void* data /* 16 bytes */)
+ {
+ XXH3_128 Xx;
+ memcpy(Xx.Hash, data, sizeof Xx);
+ return Xx;
+ }
+
+ static inline XXH3_128 HashMemory(const void* data, size_t byteCount)
+ {
+ XXH3_128 Hash;
+ XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits(data, byteCount));
+ return Hash;
+ }
+ static XXH3_128 HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); }
+ static XXH3_128 FromHexString(const char* string);
+ static XXH3_128 FromHexString(const std::string_view string);
+ const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static const int StringLength = 32;
+ typedef char String_t[StringLength + 1];
+
+ static XXH3_128 Zero; // Initialized to all zeros
+
+ inline auto operator<=>(const XXH3_128& rhs) const = default;
+
+ struct Hasher
+ {
+ size_t operator()(const XXH3_128& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct XXH3_128Stream
+{
+ /// Begin streaming hash compute (not needed on freshly constructed instance)
+ void Reset() { memset(&m_State, 0, sizeof m_State); }
+
+ /// Append another chunk
+ XXH3_128Stream& Append(const void* Data, size_t ByteCount)
+ {
+ XXH3_128bits_update(&m_State, Data, ByteCount);
+ return *this;
+ }
+
+ /// Append another chunk
+ XXH3_128Stream& Append(MemoryView Data) { return Append(Data.GetData(), Data.GetSize()); }
+
+ /// Obtain final hash. If you wish to reuse the instance call reset()
+ XXH3_128 GetHash()
+ {
+ XXH3_128 Hash;
+ XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits_digest(&m_State));
+ return Hash;
+ }
+
+private:
+ XXH3_state_s m_State{};
+};
+
+} // namespace zen
diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h
new file mode 100644
index 000000000..4a448776b
--- /dev/null
+++ b/zencore/include/zencore/zencore.h
@@ -0,0 +1,134 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <cinttypes>
+#include <exception>
+#include <string>
+
+//////////////////////////////////////////////////////////////////////////
+// Platform
+//
+
+#define ZEN_PLATFORM_WINDOWS 1
+#define ZEN_PLATFORM_LINUX 0
+#define ZEN_PLATFORM_MACOS 0
+
+//////////////////////////////////////////////////////////////////////////
+// Compiler
+//
+
+#ifdef _MSC_VER
+# define ZEN_COMPILER_MSC 1
+#endif
+
+#ifndef ZEN_COMPILER_MSC
+# define ZEN_COMPILER_MSC 0
+#endif
+
+#ifndef ZEN_COMPILER_CLANG
+# define ZEN_COMPILER_CLANG 0
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+// Build flavor
+//
+
+#ifdef NDEBUG
+# define ZEN_BUILD_DEBUG 0
+# define ZEN_BUILD_RELEASE 1
+#else
+# define ZEN_BUILD_DEBUG 1
+# define ZEN_BUILD_RELEASE 0
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+#define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1
+
+//////////////////////////////////////////////////////////////////////////
+// Assert
+//
+
+namespace zen {
+
+class AssertException : public std::exception
+{
+public:
+ AssertException(const char* Msg) : m_Msg(Msg) {}
+
+ [[nodiscard]] virtual char const* what() const override { return m_Msg.c_str(); }
+
+private:
+ std::string m_Msg;
+};
+
+} // namespace zen
+
+#define ZEN_ASSERT(x, ...) \
+ do \
+ { \
+ if (x) \
+ break; \
+ throw ::zen::AssertException{#x}; \
+ } while (false)
+
+#ifndef NDEBUG
+# define ZEN_ASSERT_SLOW(x, ...) \
+ do \
+ { \
+ if (x) \
+ break; \
+ throw ::zen::AssertException{#x}; \
+ } while (false)
+#else
+# define ZEN_ASSERT_SLOW(x, ...)
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+#ifdef __clang__
+template<typename T>
+auto ZenArrayCountHelper(T& t) -> typename std::enable_if<__is_array(T), char (&)[sizeof(t) / sizeof(t[0]) + 1]>::Type;
+#else
+template<typename T, uint32_t N>
+char (&ZenArrayCountHelper(const T (&)[N]))[N + 1];
+#endif
+
+#define ZEN_ARRAY_COUNT(array) (sizeof(ZenArrayCountHelper(array)) - 1)
+
+//////////////////////////////////////////////////////////////////////////
+
+#define ZEN_NOINLINE __declspec(noinline)
+#define ZEN_UNUSED(...) ((void)__VA_ARGS__)
+#define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false)
+#define ZENCORE_API // Placeholder to allow DLL configs in the future
+
+ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is within the stack of the currently executing thread
+ZENCORE_API bool IsApplicationExitRequested();
+ZENCORE_API void RequestApplicationExit(int ExitCode);
+
+ZENCORE_API void zencore_forcelinktests();
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_COMPILER_MSC
+# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off))
+# define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on))
+#else
+#endif
+
+// Set up optimization control macros, now that we have both the build settings and the platform macros
+#define ZEN_DISABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL
+
+#if ZEN_BUILD_DEBUG
+# define ZEN_ENABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL
+#else
+# define ZEN_ENABLE_OPTIMIZATION ZEN_ENABLE_OPTIMIZATION_ACTUAL
+#endif
+
+#define ZEN_ENABLE_OPTIMIZATION_ALWAYS ZEN_ENABLE_OPTIMIZATION_ACTUAL
+
+//////////////////////////////////////////////////////////////////////////
+
+using ThreadId_t = uint32_t;