diff options
| author | Stefan Boberg <[email protected]> | 2023-05-02 10:01:47 +0200 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-05-02 10:01:47 +0200 |
| commit | 075d17f8ada47e990fe94606c3d21df409223465 (patch) | |
| tree | e50549b766a2f3c354798a54ff73404217b4c9af /src/zencore/include | |
| parent | fix: bundle shouldn't append content zip to zen (diff) | |
| download | zen-075d17f8ada47e990fe94606c3d21df409223465.tar.xz zen-075d17f8ada47e990fe94606c3d21df409223465.zip | |
moved source directories into `/src` (#264)
* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees
Diffstat (limited to 'src/zencore/include')
46 files changed, 8890 insertions, 0 deletions
diff --git a/src/zencore/include/zencore/atomic.h b/src/zencore/include/zencore/atomic.h new file mode 100644 index 000000000..bf549e21d --- /dev/null +++ b/src/zencore/include/zencore/atomic.h @@ -0,0 +1,74 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#if ZEN_COMPILER_MSC +# include <intrin.h> +#else +# include <atomic> +#endif + +#include <cinttypes> + +namespace zen { + +inline uint32_t +AtomicIncrement(volatile uint32_t& value) +{ +#if ZEN_COMPILER_MSC + return _InterlockedIncrement((long volatile*)&value); +#else + return ((std::atomic<uint32_t>*)(&value))->fetch_add(1, std::memory_order_seq_cst) + 1; +#endif +} +inline uint32_t +AtomicDecrement(volatile uint32_t& value) +{ +#if ZEN_COMPILER_MSC + return _InterlockedDecrement((long volatile*)&value); +#else + return ((std::atomic<uint32_t>*)(&value))->fetch_sub(1, std::memory_order_seq_cst) - 1; +#endif +} + +inline uint64_t +AtomicIncrement(volatile uint64_t& value) +{ +#if ZEN_COMPILER_MSC + return _InterlockedIncrement64((__int64 volatile*)&value); +#else + return ((std::atomic<uint64_t>*)(&value))->fetch_add(1, std::memory_order_seq_cst) + 1; +#endif +} +inline uint64_t +AtomicDecrement(volatile uint64_t& value) +{ +#if ZEN_COMPILER_MSC + return _InterlockedDecrement64((__int64 volatile*)&value); +#else + return ((std::atomic<uint64_t>*)(&value))->fetch_sub(1, std::memory_order_seq_cst) - 1; +#endif +} + +inline uint32_t +AtomicAdd(volatile uint32_t& value, uint32_t amount) +{ +#if ZEN_COMPILER_MSC + return _InterlockedExchangeAdd((long volatile*)&value, amount); +#else + return ((std::atomic<uint32_t>*)(&value))->fetch_add(amount, std::memory_order_seq_cst); +#endif +} +inline uint64_t +AtomicAdd(volatile uint64_t& value, uint64_t amount) +{ +#if ZEN_COMPILER_MSC + return _InterlockedExchangeAdd64((__int64 volatile*)&value, amount); +#else + return ((std::atomic<uint64_t>*)(&value))->fetch_add(amount, std::memory_order_seq_cst); +#endif +} + +} // namespace zen diff --git a/src/zencore/include/zencore/base64.h b/src/zencore/include/zencore/base64.h new file mode 100644 index 000000000..4d78b085f --- /dev/null +++ b/src/zencore/include/zencore/base64.h @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +namespace zen { + +struct Base64 +{ + template<typename CharType> + static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest); + + static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; } +}; + +} // namespace zen diff --git a/src/zencore/include/zencore/blake3.h b/src/zencore/include/zencore/blake3.h new file mode 100644 index 000000000..b31b710a7 --- /dev/null +++ b/src/zencore/include/zencore/blake3.h @@ -0,0 +1,62 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <cinttypes> +#include <compare> +#include <cstring> + +#include <zencore/memory.h> + +namespace zen { + +class CompositeBuffer; +class StringBuilderBase; + +/** + * BLAKE3 hash - 256 bits + */ +struct BLAKE3 +{ + uint8_t Hash[32]; + + inline auto operator<=>(const BLAKE3& Rhs) const = default; + + static BLAKE3 HashBuffer(const CompositeBuffer& Buffer); + static BLAKE3 HashMemory(const void* Data, size_t ByteCount); + static BLAKE3 FromHexString(const char* String); + const char* ToHexString(char* OutString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& OutBuilder) const; + + static const int StringLength = 64; + typedef char String_t[StringLength + 1]; + + static BLAKE3 Zero; // Initialized to all zeroes + + struct Hasher + { + size_t operator()(const BLAKE3& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct BLAKE3Stream +{ + BLAKE3Stream(); + + void Reset(); // Begin streaming hash compute (not needed on freshly constructed instance) + BLAKE3Stream& Append(const void* data, size_t byteCount); // Append another chunk + BLAKE3Stream& Append(MemoryView DataView) { return Append(DataView.GetData(), DataView.GetSize()); } // Append another chunk + BLAKE3 GetHash(); // Obtain final hash. If you wish to reuse the instance call reset() + +private: + alignas(16) uint8_t m_HashState[2048]; +}; + +void blake3_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/blockingqueue.h b/src/zencore/include/zencore/blockingqueue.h new file mode 100644 index 000000000..f92df5a54 --- /dev/null +++ b/src/zencore/include/zencore/blockingqueue.h @@ -0,0 +1,76 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <atomic> +#include <condition_variable> +#include <deque> +#include <mutex> + +namespace zen { + +template<typename T> +class BlockingQueue +{ +public: + BlockingQueue() = default; + + ~BlockingQueue() { CompleteAdding(); } + + void Enqueue(T&& Item) + { + { + std::lock_guard Lock(m_Lock); + m_Queue.emplace_back(std::move(Item)); + m_Size++; + } + + m_NewItemSignal.notify_one(); + } + + bool WaitAndDequeue(T& Item) + { + if (m_CompleteAdding.load()) + { + return false; + } + + std::unique_lock Lock(m_Lock); + m_NewItemSignal.wait(Lock, [this]() { return !m_Queue.empty() || m_CompleteAdding.load(); }); + + if (!m_Queue.empty()) + { + Item = std::move(m_Queue.front()); + m_Queue.pop_front(); + m_Size--; + + return true; + } + + return false; + } + + void CompleteAdding() + { + if (!m_CompleteAdding.load()) + { + m_CompleteAdding.store(true); + m_NewItemSignal.notify_all(); + } + } + + std::size_t Size() const + { + std::unique_lock Lock(m_Lock); + return m_Queue.size(); + } + +private: + mutable std::mutex m_Lock; + std::condition_variable m_NewItemSignal; + std::deque<T> m_Queue; + std::atomic_bool m_CompleteAdding{false}; + std::atomic_uint32_t m_Size; +}; + +} // namespace zen diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h new file mode 100644 index 000000000..b546f97aa --- /dev/null +++ b/src/zencore/include/zencore/compactbinary.h @@ -0,0 +1,1475 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/enumflags.h> +#include <zencore/intmath.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/memory.h> +#include <zencore/meta.h> +#include <zencore/sharedbuffer.h> +#include <zencore/uid.h> +#include <zencore/varint.h> + +#include <functional> +#include <memory> +#include <string> +#include <string_view> +#include <type_traits> +#include <vector> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +class CbObjectView; +class CbArrayView; +class BinaryReader; +class BinaryWriter; +class CompressedBuffer; +class CbValue; + +class DateTime +{ +public: + explicit DateTime(uint64_t InTicks) : Ticks(InTicks) {} + inline DateTime(int Year, int Month, int Day, int Hours = 0, int Minutes = 0, int Seconds = 0, int MilliSeconds = 0) + { + Set(Year, Month, Day, Hours, Minutes, Seconds, MilliSeconds); + } + + inline uint64_t GetTicks() const { return Ticks; } + + static uint64_t NowTicks(); + static DateTime Now(); + + int GetYear() const; + int GetMonth() const; + int GetDay() const; + int GetHour() const; + int GetHour12() const; + int GetMinute() const; + int GetSecond() const; + int GetMillisecond() const; + void GetDate(int& Year, int& Month, int& Day) const; + + inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; } + inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; } + + std::string ToString(const char* Format) const; + std::string ToIso8601() const; + +private: + void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond); + uint64_t Ticks; // 1 tick == 0.1us == 100ns, epoch == Jan 1st 0001 +}; + +class TimeSpan +{ +public: + explicit TimeSpan(uint64_t InTicks) : Ticks(InTicks) {} + inline TimeSpan(int Hours, int Minutes, int Seconds) { Set(0, Hours, Minutes, Seconds, 0); } + inline TimeSpan(int Days, int Hours, int Minutes, int Seconds) { Set(Days, Hours, Minutes, Seconds, 0); } + inline TimeSpan(int Days, int Hours, int Minutes, int Seconds, int Nanos) { Set(Days, Hours, Minutes, Seconds, Nanos); } + + inline uint64_t GetTicks() const { return Ticks; } + inline bool operator==(const TimeSpan& Rhs) const { return Ticks == Rhs.Ticks; } + inline auto operator<=>(const TimeSpan& Rhs) const { return Ticks - Rhs.Ticks; } + + /** + * Time span related constants. + */ + + /** The maximum number of ticks that can be represented in FTimespan. */ + static constexpr int64_t MaxTicks = 9223372036854775807; + + /** The minimum number of ticks that can be represented in FTimespan. */ + static constexpr int64_t MinTicks = -9223372036854775807 - 1; + + /** The number of nanoseconds per tick. */ + static constexpr int64_t NanosecondsPerTick = 100; + + /** The number of timespan ticks per day. */ + static constexpr int64_t TicksPerDay = 864000000000; + + /** The number of timespan ticks per hour. */ + static constexpr int64_t TicksPerHour = 36000000000; + + /** The number of timespan ticks per microsecond. */ + static constexpr int64_t TicksPerMicrosecond = 10; + + /** The number of timespan ticks per millisecond. */ + static constexpr int64_t TicksPerMillisecond = 10000; + + /** The number of timespan ticks per minute. */ + static constexpr int64_t TicksPerMinute = 600000000; + + /** The number of timespan ticks per second. */ + static constexpr int64_t TicksPerSecond = 10000000; + + /** The number of timespan ticks per week. */ + static constexpr int64_t TicksPerWeek = 6048000000000; + + /** The number of timespan ticks per year (365 days, not accounting for leap years). */ + static constexpr int64_t TicksPerYear = 365 * TicksPerDay; + + int GetFractionTicks() const { return (int)(Ticks % TicksPerSecond); } + + int GetFractionMicro() const { return (int)((Ticks % TicksPerSecond) / TicksPerMicrosecond); } + + int GetFractionMilli() const { return (int)((Ticks % TicksPerSecond) / TicksPerMillisecond); } + + int GetFractionNano() const { return (int)((Ticks % TicksPerSecond) * NanosecondsPerTick); } + + int GetDays() const { return (int)(Ticks / TicksPerDay); } + + int GetHours() const { return (int)((Ticks / TicksPerHour) % 24); } + + int GetMinutes() const { return (int)((Ticks / TicksPerMinute) % 60); } + + int GetSeconds() const { return (int)((Ticks / TicksPerSecond) % 60); } + + ZENCORE_API std::string ToString(const char* Format) const; + ZENCORE_API std::string ToString() const; + +private: + void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano); + + uint64_t Ticks; +}; + +struct Guid +{ + uint32_t A, B, C, D; + + StringBuilderBase& ToString(StringBuilderBase& OutString) const; +}; + +////////////////////////////////////////////////////////////////////////// + +/** + * Field types and flags for CbField. + * + * This is a private type and is only declared here to enable inline use below. + * + * DO NOT CHANGE THE VALUE OF ANY MEMBERS OF THIS ENUM! + * BACKWARD COMPATIBILITY REQUIRES THAT THESE VALUES BE FIXED! + * SERIALIZATION USES HARD-CODED CONSTANTS BASED ON THESE VALUES! + */ +enum class CbFieldType : uint8_t +{ + /** A field type that does not occur in a valid object. */ + None = 0x00, + + /** Null. Payload is empty. */ + Null = 0x01, + + /** + * Object is an array of fields with unique non-empty names. + * + * Payload is a VarUInt byte count for the encoded fields followed by the fields. + */ + Object = 0x02, + /** + * UniformObject is an array of fields with the same field types and unique non-empty names. + * + * Payload is a VarUInt byte count for the encoded fields followed by the fields. + */ + UniformObject = 0x03, + + /** + * Array is an array of fields with no name that may be of different types. + * + * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by the fields. + */ + Array = 0x04, + /** + * UniformArray is an array of fields with no name and with the same field type. + * + * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by field type, + * followed by the fields without their field type. + */ + UniformArray = 0x05, + + /** Binary. Payload is a VarUInt byte count followed by the data. */ + Binary = 0x06, + + /** String in UTF-8. Payload is a VarUInt byte count then an unterminated UTF-8 string. */ + String = 0x07, + + /** + * Non-negative integer with the range of a 64-bit unsigned integer. + * + * Payload is the value encoded as a VarUInt. + */ + IntegerPositive = 0x08, + /** + * Negative integer with the range of a 64-bit signed integer. + * + * Payload is the ones' complement of the value encoded as a VarUInt. + */ + IntegerNegative = 0x09, + + /** Single precision float. Payload is one big endian IEEE 754 binary32 float. */ + Float32 = 0x0a, + /** Double precision float. Payload is one big endian IEEE 754 binary64 float. */ + Float64 = 0x0b, + + /** Boolean false value. Payload is empty. */ + BoolFalse = 0x0c, + /** Boolean true value. Payload is empty. */ + BoolTrue = 0x0d, + + /** + * ObjectAttachment is a reference to a compact binary attachment stored externally. + * + * Payload is a 160-bit hash digest of the referenced compact binary data. + */ + ObjectAttachment = 0x0e, + /** + * BinaryAttachment is a reference to a binary attachment stored externally. + * + * Payload is a 160-bit hash digest of the referenced binary data. + */ + BinaryAttachment = 0x0f, + + /** Hash. Payload is a 160-bit hash digest. */ + Hash = 0x10, + /** UUID/GUID. Payload is a 128-bit UUID as defined by RFC 4122. */ + Uuid = 0x11, + + /** + * Date and time between 0001-01-01 00:00:00.0000000 and 9999-12-31 23:59:59.9999999. + * + * Payload is a big endian int64 count of 100ns ticks since 0001-01-01 00:00:00.0000000. + */ + DateTime = 0x12, + /** + * Difference between two date/time values. + * + * Payload is a big endian int64 count of 100ns ticks in the span, and may be negative. + */ + TimeSpan = 0x13, + + /** + * Object ID + * + * Payload is a 12-byte opaque identifier + */ + ObjectId = 0x14, + + /** + * CustomById identifies the sub-type of its payload by an integer identifier. + * + * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed + * by a VarUInt of the sub-type identifier then the payload of the sub-type. + */ + CustomById = 0x1e, + /** + * CustomByType identifies the sub-type of its payload by a string identifier. + * + * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed + * by a VarUInt byte count of the unterminated sub-type identifier, then the sub-type identifier + * without termination, then the payload of the sub-type. + */ + CustomByName = 0x1f, + + /** Reserved for future use as a flag. Do not add types in this range. */ + Reserved = 0x20, + + /** + * A transient flag which indicates that the object or array containing this field has stored + * the field type before the payload and name. Non-uniform objects and fields will set this. + * + * Note: Since the flag must never be serialized, this bit may be repurposed in the future. + */ + HasFieldType = 0x40, + + /** A persisted flag which indicates that the field has a name stored before the payload. */ + HasFieldName = 0x80, +}; + +ENUM_CLASS_FLAGS(CbFieldType); + +/** Functions that operate on CbFieldType. */ +class CbFieldTypeOps +{ + static constexpr CbFieldType SerializedTypeMask = CbFieldType(0b1011'1111); + static constexpr CbFieldType TypeMask = CbFieldType(0b0011'1111); + static constexpr CbFieldType ObjectMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType ObjectBase = CbFieldType(0b0000'0010); + static constexpr CbFieldType ArrayMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType ArrayBase = CbFieldType(0b0000'0100); + static constexpr CbFieldType IntegerMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType IntegerBase = CbFieldType(0b0000'1000); + static constexpr CbFieldType FloatMask = CbFieldType(0b0011'1100); + static constexpr CbFieldType FloatBase = CbFieldType(0b0000'1000); + static constexpr CbFieldType BoolMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType BoolBase = CbFieldType(0b0000'1100); + static constexpr CbFieldType AttachmentMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType AttachmentBase = CbFieldType(0b0000'1110); + + static void StaticAssertTypeConstants(); + +public: + /** The type with flags removed. */ + static constexpr inline CbFieldType GetType(CbFieldType Type) { return Type & TypeMask; } + /** The type with transient flags removed. */ + static constexpr inline CbFieldType GetSerializedType(CbFieldType Type) { return Type & SerializedTypeMask; } + + static constexpr inline bool HasFieldType(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldType); } + static constexpr inline bool HasFieldName(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldName); } + + static constexpr inline bool IsNone(CbFieldType Type) { return GetType(Type) == CbFieldType::None; } + static constexpr inline bool IsNull(CbFieldType Type) { return GetType(Type) == CbFieldType::Null; } + + static constexpr inline bool IsObject(CbFieldType Type) { return (Type & ObjectMask) == ObjectBase; } + static constexpr inline bool IsArray(CbFieldType Type) { return (Type & ArrayMask) == ArrayBase; } + + static constexpr inline bool IsBinary(CbFieldType Type) { return GetType(Type) == CbFieldType::Binary; } + static constexpr inline bool IsString(CbFieldType Type) { return GetType(Type) == CbFieldType::String; } + + static constexpr inline bool IsInteger(CbFieldType Type) { return (Type & IntegerMask) == IntegerBase; } + /** Whether the field is a float, or integer due to implicit conversion. */ + static constexpr inline bool IsFloat(CbFieldType Type) { return (Type & FloatMask) == FloatBase; } + static constexpr inline bool IsBool(CbFieldType Type) { return (Type & BoolMask) == BoolBase; } + + static constexpr inline bool IsObjectAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectAttachment; } + static constexpr inline bool IsBinaryAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::BinaryAttachment; } + static constexpr inline bool IsAttachment(CbFieldType Type) { return (Type & AttachmentMask) == AttachmentBase; } + + static constexpr inline bool IsHash(CbFieldType Type) + { + switch (GetType(Type)) + { + case CbFieldType::Hash: + case CbFieldType::BinaryAttachment: + case CbFieldType::ObjectAttachment: + return true; + default: + return false; + } + } + + static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; } + static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; } + + static constexpr inline bool IsCustomById(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomById; } + static constexpr inline bool IsCustomByName(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomByName; } + + static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; } + static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; } + + /** Whether the type is or may contain fields of any attachment type. */ + static constexpr inline bool MayContainAttachments(CbFieldType Type) + { + return int(IsObject(Type) == true) | int(IsArray(Type) == true) | int(IsAttachment(Type) == true); + } +}; + +/** Errors that can occur when accessing a field. */ +enum class CbFieldError : uint8_t +{ + /** The field is not in an error state. */ + None, + /** The value type does not match the requested type. */ + TypeError, + /** The value is out of range for the requested type. */ + RangeError, +}; + +class ICbVisitor +{ +public: + virtual void SetName(std::string_view Name) = 0; + virtual void BeginObject() = 0; + virtual void EndObject() = 0; + virtual void BeginArray() = 0; + virtual void EndArray() = 0; + virtual void VisitNull() = 0; + virtual void VisitBinary(SharedBuffer Value) = 0; + virtual void VisitString(std::string_view Value) = 0; + virtual void VisitInteger(int64_t Value) = 0; + virtual void VisitInteger(uint64_t Value) = 0; + virtual void VisitFloat(float Value) = 0; + virtual void VisitDouble(double Value) = 0; + virtual void VisitBool(bool value) = 0; + virtual void VisitCbAttachment(const IoHash& Value) = 0; + virtual void VisitBinaryAttachment(const IoHash& Value) = 0; + virtual void VisitHash(const IoHash& Value) = 0; + virtual void VisitUuid(const Guid& Value) = 0; + virtual void VisitObjectId(const Oid& Value) = 0; + virtual void VisitDateTime(DateTime Value) = 0; + virtual void VisitTimeSpan(TimeSpan Value) = 0; +}; + +/** A custom compact binary field type with an integer identifier. */ +struct CbCustomById +{ + /** An identifier for the sub-type of the field. */ + uint64_t Id = 0; + /** A view of the value. Lifetime is tied to the field that the value is associated with. */ + MemoryView Data; +}; + +/** A custom compact binary field type with a string identifier. */ +struct CbCustomByName +{ + /** An identifier for the sub-type of the field. Lifetime is tied to the field that the name is associated with. */ + std::u8string_view Name; + /** A view of the value. Lifetime is tied to the field that the value is associated with. */ + MemoryView Data; +}; + +namespace CompactBinaryPrivate { + /** Parameters for converting to an integer. */ + struct IntegerParams + { + /** Whether the output type has a sign bit. */ + uint32_t IsSigned : 1; + /** Bits of magnitude. (7 for int8) */ + uint32_t MagnitudeBits : 31; + }; + + /** Make integer params for the given integer type. */ + template<typename IntType> + static constexpr inline IntegerParams MakeIntegerParams() + { + IntegerParams Params; + Params.IsSigned = IntType(-1) < IntType(0); + Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned; + return Params; + } + +} // namespace CompactBinaryPrivate + +/** + * An atom of data in the compact binary format. + * + * Accessing the value of a field is always a safe operation, even if accessed as the wrong type. + * An invalid access will return a default value for the requested type, and set an error code on + * the field that can be checked with GetLastError and HasLastError. A valid access will clear an + * error from a previous invalid access. + * + * A field is encoded in one or more bytes, depending on its type and the type of object or array + * that contains it. A field of an object or array which is non-uniform encodes its field type in + * the first byte, and includes the HasFieldName flag for a field in an object. The field name is + * encoded in a variable-length unsigned integer of its size in bytes, for named fields, followed + * by that many bytes of the UTF-8 encoding of the name with no null terminator. The remainder of + * the field is the payload and is described in the field type enum. Every field must be uniquely + * addressable when encoded, which means a zero-byte field is not permitted, and only arises in a + * uniform array of fields with no payload, where the answer is to encode as a non-uniform array. + * + * This type only provides a view into memory and does not perform any memory management itself. + * Use CbFieldRef to hold a reference to the underlying memory when necessary. + */ + +class CbFieldView +{ +public: + CbFieldView() = default; + + ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType); + + /** Construct a field from a value, without access to the name. */ + inline explicit CbFieldView(const CbValue& Value); + + /** Returns the name of the field if it has a name, otherwise an empty view. */ + constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); } + /** Returns the name of the field if it has a name, otherwise an empty view. */ + constexpr inline std::u8string_view GetU8Name() const + { + return std::u8string_view(static_cast<const char8_t*>(Payload) - NameLen, NameLen); + } + + /** Returns the value for unchecked access. Prefer the typed accessors below. */ + inline CbValue GetValue() const; + + ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView()); + ZENCORE_API CbObjectView AsObjectView(); + ZENCORE_API CbArrayView AsArrayView(); + ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view()); + ZENCORE_API std::u8string_view AsU8String(std::u8string_view Default = std::u8string_view()); + + ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const; + + /** Access the field as an int8. Returns the provided default on error. */ + inline int8_t AsInt8(int8_t Default = 0) { return AsInteger<int8_t>(Default); } + /** Access the field as an int16. Returns the provided default on error. */ + inline int16_t AsInt16(int16_t Default = 0) { return AsInteger<int16_t>(Default); } + /** Access the field as an int32. Returns the provided default on error. */ + inline int32_t AsInt32(int32_t Default = 0) { return AsInteger<int32_t>(Default); } + /** Access the field as an int64. Returns the provided default on error. */ + inline int64_t AsInt64(int64_t Default = 0) { return AsInteger<int64_t>(Default); } + /** Access the field as a uint8. Returns the provided default on error. */ + inline uint8_t AsUInt8(uint8_t Default = 0) { return AsInteger<uint8_t>(Default); } + /** Access the field as a uint16. Returns the provided default on error. */ + inline uint16_t AsUInt16(uint16_t Default = 0) { return AsInteger<uint16_t>(Default); } + /** Access the field as a uint32. Returns the provided default on error. */ + inline uint32_t AsUInt32(uint32_t Default = 0) { return AsInteger<uint32_t>(Default); } + /** Access the field as a uint64. Returns the provided default on error. */ + inline uint64_t AsUInt64(uint64_t Default = 0) { return AsInteger<uint64_t>(Default); } + + /** Access the field as a float. Returns the provided default on error. */ + ZENCORE_API float AsFloat(float Default = 0.0f); + /** Access the field as a double. Returns the provided default on error. */ + ZENCORE_API double AsDouble(double Default = 0.0); + + /** Access the field as a bool. Returns the provided default on error. */ + ZENCORE_API bool AsBool(bool bDefault = false); + + /** Access the field as a hash referencing a compact binary attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsObjectAttachment(const IoHash& Default = IoHash()); + /** Access the field as a hash referencing a binary attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsBinaryAttachment(const IoHash& Default = IoHash()); + /** Access the field as a hash referencing an attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsAttachment(const IoHash& Default = IoHash()); + + /** Access the field as a hash. Returns the provided default on error. */ + ZENCORE_API IoHash AsHash(const IoHash& Default = IoHash()); + + /** Access the field as a UUID. Returns a nil UUID on error. */ + ZENCORE_API Guid AsUuid(); + /** Access the field as a UUID. Returns the provided default on error. */ + ZENCORE_API Guid AsUuid(const Guid& Default); + + /** Access the field as an OID. Returns a nil OID on error. */ + ZENCORE_API Oid AsObjectId(); + /** Access the field as a OID. Returns the provided default on error. */ + ZENCORE_API Oid AsObjectId(const Oid& Default); + + /** Access the field as a custom sub-type with an integer identifier. Returns the provided default on error. */ + ZENCORE_API CbCustomById AsCustomById(CbCustomById Default); + /** Access the field as a custom sub-type with a string identifier. Returns the provided default on error. */ + ZENCORE_API CbCustomByName AsCustomByName(CbCustomByName Default); + + /** Access the field as a date/time tick count. Returns the provided default on error. */ + ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0); + + /** Access the field as a date/time. Returns a date/time at the epoch on error. */ + ZENCORE_API DateTime AsDateTime(); + /** Access the field as a date/time. Returns the provided default on error. */ + ZENCORE_API DateTime AsDateTime(DateTime Default); + + /** Access the field as a timespan tick count. Returns the provided default on error. */ + ZENCORE_API int64_t AsTimeSpanTicks(int64_t Default = 0); + + /** Access the field as a timespan. Returns an empty timespan on error. */ + ZENCORE_API TimeSpan AsTimeSpan(); + /** Access the field as a timespan. Returns the provided default on error. */ + ZENCORE_API TimeSpan AsTimeSpan(TimeSpan Default); + + /** True if the field has a name. */ + constexpr inline bool HasName() const { return CbFieldTypeOps::HasFieldName(Type); } + + constexpr inline bool IsNull() const { return CbFieldTypeOps::IsNull(Type); } + + constexpr inline bool IsObject() const { return CbFieldTypeOps::IsObject(Type); } + constexpr inline bool IsArray() const { return CbFieldTypeOps::IsArray(Type); } + + constexpr inline bool IsBinary() const { return CbFieldTypeOps::IsBinary(Type); } + constexpr inline bool IsString() const { return CbFieldTypeOps::IsString(Type); } + + /** Whether the field is an integer of unspecified range and sign. */ + constexpr inline bool IsInteger() const { return CbFieldTypeOps::IsInteger(Type); } + /** Whether the field is a float, or integer that supports implicit conversion. */ + constexpr inline bool IsFloat() const { return CbFieldTypeOps::IsFloat(Type); } + constexpr inline bool IsBool() const { return CbFieldTypeOps::IsBool(Type); } + + constexpr inline bool IsObjectAttachment() const { return CbFieldTypeOps::IsObjectAttachment(Type); } + constexpr inline bool IsBinaryAttachment() const { return CbFieldTypeOps::IsBinaryAttachment(Type); } + constexpr inline bool IsAttachment() const { return CbFieldTypeOps::IsAttachment(Type); } + + constexpr inline bool IsHash() const { return CbFieldTypeOps::IsHash(Type); } + constexpr inline bool IsUuid() const { return CbFieldTypeOps::IsUuid(Type); } + constexpr inline bool IsObjectId() const { return CbFieldTypeOps::IsObjectId(Type); } + + constexpr inline bool IsDateTime() const { return CbFieldTypeOps::IsDateTime(Type); } + constexpr inline bool IsTimeSpan() const { return CbFieldTypeOps::IsTimeSpan(Type); } + + /** Whether the field has a value. */ + constexpr inline explicit operator bool() const { return HasValue(); } + + /** + * Whether the field has a value. + * + * All fields in a valid object or array have a value. A field with no value is returned when + * finding a field by name fails or when accessing an iterator past the end. + */ + constexpr inline bool HasValue() const { return !CbFieldTypeOps::IsNone(Type); }; + + /** Whether the last field access encountered an error. */ + constexpr inline bool HasError() const { return Error != CbFieldError::None; } + + /** The type of error that occurred on the last field access, or None. */ + constexpr inline CbFieldError GetError() const { return Error; } + + /** Returns the size of the field in bytes, including the type and name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the field, including the type and name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& HashStream) const; + + /** Feed the field (including type and name) to the stream function */ + inline void WriteToStream(auto Hash) const + { + const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type); + Hash(&SerializedType, sizeof(SerializedType)); + auto View = GetViewNoType(); + Hash(View.GetData(), View.GetSize()); + } + + /** Copy the field into a buffer of exactly GetSize() bytes, including the type and name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the field into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + /** + * Whether this field is identical to the other field. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be performed with ValidateCompactBinary, except for field order and field name case. + */ + ZENCORE_API bool Equals(const CbFieldView& Other) const; + + /** Returns a view of the field, including the type and name when present. */ + ZENCORE_API MemoryView GetView() const; + + /** + * Try to get a view of the field as it would be serialized, such as by CopyTo. + * + * A serialized view is not available if the field has an externally-provided type. + * Access the serialized form of such fields using CopyTo or FCbFieldRef::Clone. + */ + inline bool TryGetSerializedView(MemoryView& OutView) const + { + if (CbFieldTypeOps::HasFieldType(Type)) + { + OutView = GetView(); + return true; + } + return false; + } + +protected: + /** Returns a view of the name and value payload, which excludes the type. */ + ZENCORE_API MemoryView GetViewNoType() const; + + /** Returns a view of the value payload, which excludes the type and name. */ + inline MemoryView GetPayloadView() const { return MemoryView(Payload, GetPayloadSize()); } + + /** Returns the type of the field including flags. */ + constexpr inline CbFieldType GetType() const { return Type; } + + /** Returns the start of the value payload. */ + constexpr inline const void* GetPayload() const { return Payload; } + + /** Returns the end of the value payload. */ + inline const void* GetPayloadEnd() const { return static_cast<const uint8_t*>(Payload) + GetPayloadSize(); } + + /** Returns the size of the value payload in bytes, which is the field excluding the type and name. */ + ZENCORE_API uint64_t GetPayloadSize() const; + + /** Assign a field from a pointer to its data and an optional externally-provided type. */ + inline void Assign(const void* InData, const CbFieldType InType) + { + static_assert(std::is_trivially_destructible<CbFieldView>::value, + "This optimization requires CbField to be trivially destructible!"); + new (this) CbFieldView(InData, InType); + } + +private: + /** + * Access the field as the given integer type. + * + * Returns the provided default if the value cannot be represented in the output type. + */ + template<typename IntType> + inline IntType AsInteger(IntType Default) + { + return IntType(AsInteger(uint64_t(Default), CompactBinaryPrivate::MakeIntegerParams<IntType>())); + } + + ZENCORE_API uint64_t AsInteger(uint64_t Default, CompactBinaryPrivate::IntegerParams Params); + +private: + /** The field type, with the transient HasFieldType flag if the field contains its type. */ + CbFieldType Type = CbFieldType::None; + /** The error (if any) that occurred on the last field access. */ + CbFieldError Error = CbFieldError::None; + /** The number of bytes for the name stored before the payload. */ + uint32_t NameLen = 0; + /** The value payload, which also points to the end of the name. */ + const void* Payload = nullptr; +}; + +template<typename FieldType> +class TCbFieldIterator : public FieldType +{ +public: + /** Construct an empty field range. */ + constexpr TCbFieldIterator() = default; + + inline TCbFieldIterator& operator++() + { + const void* const PayloadEnd = FieldType::GetPayloadEnd(); + const int64_t AtEndMask = int64_t(PayloadEnd == FieldsEnd) - 1; + const CbFieldType NextType = CbFieldType(int64_t(FieldType::GetType()) & AtEndMask); + const void* const NextField = reinterpret_cast<const void*>(int64_t(PayloadEnd) & AtEndMask); + const void* const NextFieldsEnd = reinterpret_cast<const void*>(int64_t(FieldsEnd) & AtEndMask); + + FieldType::Assign(NextField, NextType); + FieldsEnd = NextFieldsEnd; + return *this; + } + + inline TCbFieldIterator operator++(int) + { + TCbFieldIterator It(*this); + ++*this; + return It; + } + + constexpr inline FieldType& operator*() { return *this; } + constexpr inline FieldType* operator->() { return this; } + + /** Reset this to an empty field range. */ + inline void Reset() { *this = TCbFieldIterator(); } + + /** Returns the size of the fields in the range in bytes. */ + ZENCORE_API uint64_t GetRangeSize() const; + + /** Calculate the hash of every field in the range. */ + ZENCORE_API IoHash GetRangeHash() const; + ZENCORE_API void GetRangeHash(IoHashStream& Hash) const; + + using FieldType::Equals; + + template<typename OtherFieldType> + constexpr inline bool Equals(const TCbFieldIterator<OtherFieldType>& Other) const + { + return FieldType::GetPayload() == Other.OtherFieldType::GetPayload() && FieldsEnd == Other.FieldsEnd; + } + + template<typename OtherFieldType> + constexpr inline bool operator==(const TCbFieldIterator<OtherFieldType>& Other) const + { + return Equals(Other); + } + + template<typename OtherFieldType> + constexpr inline bool operator!=(const TCbFieldIterator<OtherFieldType>& Other) const + { + return !Equals(Other); + } + + /** Copy the field range into a buffer of exactly GetRangeSize() bytes. */ + ZENCORE_API void CopyRangeTo(MutableMemoryView Buffer) const; + + /** Invoke the visitor for every attachment in the field range. */ + ZENCORE_API void IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const; + + /** Create a view of every field in the range. */ + inline MemoryView GetRangeView() const { return MemoryView(FieldType::GetView().GetData(), FieldsEnd); } + + /** + * Try to get a view of every field in the range as they would be serialized. + * + * A serialized view is not available if the underlying fields have an externally-provided type. + * Access the serialized form of such ranges using CbFieldRefIterator::CloneRange. + */ + inline bool TryGetSerializedRangeView(MemoryView& OutView) const + { + if (CbFieldTypeOps::HasFieldType(FieldType::GetType())) + { + OutView = GetRangeView(); + return true; + } + return false; + } + +protected: + /** Construct a field range that contains exactly one field. */ + constexpr inline explicit TCbFieldIterator(FieldType InField) : FieldType(std::move(InField)), FieldsEnd(FieldType::GetPayloadEnd()) {} + + /** + * Construct a field range from the first field and a pointer to the end of the last field. + * + * @param InField The first field, or the default field if there are no fields. + * @param InFieldsEnd A pointer to the end of the payload of the last field, or null. + */ + constexpr inline TCbFieldIterator(FieldType&& InField, const void* InFieldsEnd) : FieldType(std::move(InField)), FieldsEnd(InFieldsEnd) + { + } + + /** Returns the end of the last field, or null for an iterator at the end. */ + template<typename OtherFieldType> + static inline const void* GetFieldsEnd(const TCbFieldIterator<OtherFieldType>& It) + { + return It.FieldsEnd; + } + +private: + friend inline TCbFieldIterator begin(const TCbFieldIterator& Iterator) { return Iterator; } + friend inline TCbFieldIterator end(const TCbFieldIterator&) { return TCbFieldIterator(); } + +private: + template<typename OtherType> + friend class TCbFieldIterator; + + friend class CbFieldViewIterator; + + friend class CbFieldIterator; + + /** Pointer to the first byte past the end of the last field. Set to null at the end. */ + const void* FieldsEnd = nullptr; +}; + +/** + * Iterator for CbField. + * + * @see CbFieldIterator + */ +class CbFieldViewIterator : public TCbFieldIterator<CbFieldView> +{ +public: + constexpr CbFieldViewIterator() = default; + + /** Construct a field range that contains exactly one field. */ + static inline CbFieldViewIterator MakeSingle(const CbFieldView& Field) { return CbFieldViewIterator(Field); } + + /** + * Construct a field range from a buffer containing zero or more valid fields. + * + * @param View A buffer containing zero or more valid fields. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + */ + static inline CbFieldViewIterator MakeRange(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType) + { + return !View.IsEmpty() ? TCbFieldIterator(CbFieldView(View.GetData(), Type), View.GetDataEnd()) : CbFieldViewIterator(); + } + + /** Construct an iterator from another iterator. */ + template<typename OtherFieldType> + inline CbFieldViewIterator(const TCbFieldIterator<OtherFieldType>& It) + : TCbFieldIterator(ImplicitConv<CbFieldView>(It), GetFieldsEnd(It)) + { + } + +private: + using TCbFieldIterator::TCbFieldIterator; +}; + +/** + * Serialize a compact binary array to JSON. + */ +ZENCORE_API void CompactBinaryToJson(const CbArrayView& Object, StringBuilderBase& Builder); + +/** + * Array of CbField that have no names. + * + * Accessing a field of the array requires iteration. Access by index is not provided because the + * cost of accessing an item by index scales linearly with the index. + * + * This type only provides a view into memory and does not perform any memory management itself. + * Use CbArrayRef to hold a reference to the underlying memory when necessary. + */ +class CbArrayView : protected CbFieldView +{ + friend class CbFieldView; + +public: + /** @see CbField::CbField */ + using CbFieldView::CbFieldView; + + /** Construct an array with no fields. */ + ZENCORE_API CbArrayView(); + + /** Returns the number of items in the array. */ + ZENCORE_API uint64_t Num() const; + + /** Create an iterator for the fields of this array. */ + ZENCORE_API CbFieldViewIterator CreateViewIterator() const; + + /** Visit the fields of this array. */ + ZENCORE_API void VisitFields(ICbVisitor& Visitor); + + /** Access the array as an array field. */ + inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } + + /** Construct an array from an array field. No type check is performed! */ + static inline CbArrayView FromFieldView(const CbFieldView& Field) { return CbArrayView(Field); } + + /** Whether the array has any fields. */ + inline explicit operator bool() const { return Num() > 0; } + + /** Returns the size of the array in bytes if serialized by itself with no name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the array if serialized by itself with no name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& Stream) const; + + /** + * Whether this array is identical to the other array. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be done with the All mode to check these assumptions about the format of the inputs. + */ + ZENCORE_API bool Equals(const CbArrayView& Other) const; + + /** Copy the array into a buffer of exactly GetSize() bytes, with no name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the array into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + ///** Invoke the visitor for every attachment in the array. */ + inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const + { + CreateViewIterator().IterateRangeAttachments(Visitor); + } + + /** Returns a view of the array, including the type and name when present. */ + using CbFieldView::GetView; + + StringBuilderBase& ToJson(StringBuilderBase& Builder) const + { + CompactBinaryToJson(*this, Builder); + return Builder; + } + +private: + friend inline CbFieldViewIterator begin(const CbArrayView& Array) { return Array.CreateViewIterator(); } + friend inline CbFieldViewIterator end(const CbArrayView&) { return CbFieldViewIterator(); } + + /** Construct an array from an array field. No type check is performed! Use via FromField. */ + inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {} +}; + +/** + * Serialize a compact binary object to JSON. + */ +ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder); + +class CbObjectView : protected CbFieldView +{ + friend class CbFieldView; + +public: + /** @see CbField::CbField */ + using CbFieldView::CbFieldView; + + using CbFieldView::TryGetSerializedView; + + /** Construct an object with no fields. */ + ZENCORE_API CbObjectView(); + + /** Create an iterator for the fields of this object. */ + ZENCORE_API CbFieldViewIterator CreateViewIterator() const; + + /** Visit the fields of this object. */ + ZENCORE_API void VisitFields(ICbVisitor& Visitor); + + /** + * Find a field by case-sensitive name comparison. + * + * The cost of this operation scales linearly with the number of fields in the object. Prefer + * to iterate over the fields only once when consuming an object. + * + * @param Name The name of the field. + * @return The matching field if found, otherwise a field with no value. + */ + ZENCORE_API CbFieldView FindView(std::string_view Name) const; + + /** Find a field by case-insensitive name comparison. */ + ZENCORE_API CbFieldView FindViewIgnoreCase(std::string_view Name) const; + + /** Find a field by case-sensitive name comparison. */ + inline CbFieldView operator[](std::string_view Name) const { return FindView(Name); } + + /** Access the object as an object field. */ + inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } + + /** Construct an object from an object field. No type check is performed! */ + static inline CbObjectView FromFieldView(const CbFieldView& Field) { return CbObjectView(Field); } + + /** Whether the object has any fields. */ + ZENCORE_API explicit operator bool() const; + + /** Returns the size of the object in bytes if serialized by itself with no name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the object if serialized by itself with no name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& HashStream) const; + + /** + * Whether this object is identical to the other object. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be done with the All mode to check these assumptions about the format of the inputs. + */ + ZENCORE_API bool Equals(const CbObjectView& Other) const; + + /** Copy the object into a buffer of exactly GetSize() bytes, with no name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the field into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + ///** Invoke the visitor for every attachment in the object. */ + inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const + { + CreateViewIterator().IterateRangeAttachments(Visitor); + } + + /** Returns a view of the object, including the type and name when present. */ + using CbFieldView::GetView; + + /** Whether the field has a value. */ + using CbFieldView::operator bool; + + StringBuilderBase& ToJson(StringBuilderBase& Builder) const + { + CompactBinaryToJson(*this, Builder); + return Builder; + } + +private: + friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); } + friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); } + + /** Construct an object from an object field. No type check is performed! Use via FromField. */ + inline explicit CbObjectView(const CbFieldView& Field) : CbFieldView(Field) {} +}; + +////////////////////////////////////////////////////////////////////////// + +/** A reference to a function that is used to allocate buffers for compact binary data. */ +using BufferAllocator = std::function<UniqueBuffer(uint64_t Size)>; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** A wrapper that holds a reference to the buffer that contains its compact binary value. */ +template<typename BaseType> +class CbBuffer : public BaseType +{ +public: + /** Construct a default value. */ + CbBuffer() = default; + + /** + * Construct a value from a pointer to its data and an optional externally-provided type. + * + * @param ValueBuffer A buffer that exactly contains the value. + * @param Type HasFieldType means that ValueBuffer contains the type. Otherwise, use the given type. + */ + inline explicit CbBuffer(SharedBuffer ValueBuffer, CbFieldType Type = CbFieldType::HasFieldType) + { + if (ValueBuffer) + { + BaseType::operator=(BaseType(ValueBuffer.GetData(), Type)); + ZEN_ASSERT(ValueBuffer.GetView().Contains(BaseType::GetView())); + Buffer = std::move(ValueBuffer); + } + } + + /** Construct a value that holds a reference to the buffer that contains it. */ + inline CbBuffer(const BaseType& Value, SharedBuffer OuterBuffer) : BaseType(Value) + { + if (OuterBuffer) + { + ZEN_ASSERT(OuterBuffer.GetView().Contains(BaseType::GetView())); + Buffer = std::move(OuterBuffer); + } + } + + /** Construct a value that holds a reference to the buffer of the outer that contains it. */ + template<typename OtherBaseType> + inline CbBuffer(const BaseType& Value, CbBuffer<OtherBaseType> OuterRef) : CbBuffer(Value, std::move(OuterRef.Buffer)) + { + } + + /** Reset this to a default value and null buffer. */ + inline void Reset() { *this = CbBuffer(); } + + /** Whether this reference has ownership of the memory in its buffer. */ + inline bool IsOwned() const { return Buffer && Buffer.IsOwned(); } + + /** Clone the value, if necessary, to a buffer that this reference has ownership of. */ + inline void MakeOwned() + { + if (!IsOwned()) + { + UniqueBuffer MutableBuffer = UniqueBuffer::Alloc(BaseType::GetSize()); + BaseType::CopyTo(MutableBuffer); + BaseType::operator=(BaseType(MutableBuffer.GetData())); + Buffer = std::move(MutableBuffer); + } + } + + /** Returns a buffer that exactly contains this value. */ + inline SharedBuffer GetBuffer() const + { + const MemoryView View = BaseType::GetView(); + const SharedBuffer& OuterBuffer = GetOuterBuffer(); + return View == OuterBuffer.GetView() ? OuterBuffer : SharedBuffer::MakeView(View, OuterBuffer); + } + + /** Returns the outer buffer (if any) that contains this value. */ + inline const SharedBuffer& GetOuterBuffer() const& { return Buffer; } + inline SharedBuffer GetOuterBuffer() && { return std::move(Buffer); } + +private: + template<typename OtherType> + friend class CbBuffer; + + SharedBuffer Buffer; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Factory functions for types derived from CbBuffer. + * + * This uses the curiously recurring template pattern to construct the correct type of reference. + * The derived type inherits from CbBufferRef and this type to expose the factory functions. + */ +template<typename RefType, typename BaseType> +class CbBufferFactory +{ +public: + /** Construct a value from an owned clone of its memory. */ + static inline RefType Clone(const void* const Data) { return Clone(BaseType(Data)); } + + /** Construct a value from an owned clone of its memory. */ + static inline RefType Clone(const BaseType& Value) + { + RefType Ref = MakeView(Value); + Ref.MakeOwned(); + return Ref; + } + + /** Construct a value from a read-only view of its memory and its optional outer buffer. */ + static inline RefType MakeView(const void* const Data, SharedBuffer OuterBuffer = SharedBuffer()) + { + return MakeView(BaseType(Data), std::move(OuterBuffer)); + } + + /** Construct a value from a read-only view of its memory and its optional outer buffer. */ + static inline RefType MakeView(const BaseType& Value, SharedBuffer OuterBuffer = SharedBuffer()) + { + return RefType(Value, std::move(OuterBuffer)); + } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class CbArray; +class CbObject; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A field that can hold a reference to the memory that contains it. + * + * @see CbBufferRef + */ +class CbField : public CbBuffer<CbFieldView>, public CbBufferFactory<CbField, CbFieldView> +{ +public: + using CbBuffer::CbBuffer; + + /** Access the field as an object. Defaults to an empty object on error. */ + inline CbObject AsObject() &; + inline CbObject AsObject() &&; + + /** Access the field as an array. Defaults to an empty array on error. */ + inline CbArray AsArray() &; + inline CbArray AsArray() &&; + + /** Access the field as binary. Returns the provided default on error. */ + inline SharedBuffer AsBinary(const SharedBuffer& Default = SharedBuffer()) &; + inline SharedBuffer AsBinary(const SharedBuffer& Default = SharedBuffer()) &&; +}; + +/** + * Iterator for CbFieldRef. + * + * @see CbFieldIterator + */ +class CbFieldIterator : public TCbFieldIterator<CbField> +{ +public: + /** Construct a field range from an owned clone of a range. */ + ZENCORE_API static CbFieldIterator CloneRange(const CbFieldViewIterator& It); + + /** Construct a field range from an owned clone of a range. */ + static inline CbFieldIterator CloneRange(const CbFieldIterator& It) { return CloneRange(CbFieldViewIterator(It)); } + + /** Construct a field range that contains exactly one field. */ + static inline CbFieldIterator MakeSingle(CbField Field) { return CbFieldIterator(std::move(Field)); } + + /** + * Construct a field range from a buffer containing zero or more valid fields. + * + * @param Buffer A buffer containing zero or more valid fields. + * @param Type HasFieldType means that Buffer contains the type. Otherwise, use the given type. + */ + static inline CbFieldIterator MakeRange(SharedBuffer Buffer, CbFieldType Type = CbFieldType::HasFieldType) + { + if (Buffer.GetSize()) + { + const void* const DataEnd = Buffer.GetView().GetDataEnd(); + return CbFieldIterator(CbField(std::move(Buffer), Type), DataEnd); + } + return CbFieldIterator(); + } + + /** Construct a field range from an iterator and its optional outer buffer. */ + static inline CbFieldIterator MakeRangeView(const CbFieldViewIterator& It, SharedBuffer OuterBuffer = SharedBuffer()) + { + return CbFieldIterator(CbField(It, std::move(OuterBuffer)), GetFieldsEnd(It)); + } + + /** Construct an empty field range. */ + constexpr CbFieldIterator() = default; + + /** Clone the range, if necessary, to a buffer that this reference has ownership of. */ + inline void MakeRangeOwned() + { + if (!IsOwned()) + { + *this = CloneRange(*this); + } + } + + /** Returns a buffer that exactly contains the field range. */ + ZENCORE_API SharedBuffer GetRangeBuffer() const; + +private: + using TCbFieldIterator::TCbFieldIterator; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An array that can hold a reference to the memory that contains it. + * + * @see CbBuffer + */ +class CbArray : public CbBuffer<CbArrayView>, public CbBufferFactory<CbArray, CbArrayView> +{ +public: + using CbBuffer::CbBuffer; + + /** Create an iterator for the fields of this array. */ + inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); } + + /** Access the array as an array field. */ + inline CbField AsField() const& { return CbField(CbArrayView::AsFieldView(), *this); } + + /** Access the array as an array field. */ + inline CbField AsField() && { return CbField(CbArrayView::AsFieldView(), std::move(*this)); } + +private: + friend inline CbFieldIterator begin(const CbArray& Array) { return Array.CreateIterator(); } + friend inline CbFieldIterator end(const CbArray&) { return CbFieldIterator(); } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An object that can hold a reference to the memory that contains it. + * + * @see CbBuffer + */ +class CbObject : public CbBuffer<CbObjectView>, public CbBufferFactory<CbObject, CbObjectView> +{ +public: + using CbBuffer::CbBuffer; + + /** Create an iterator for the fields of this object. */ + inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); } + + /** Find a field by case-sensitive name comparison. */ + inline CbField Find(std::string_view Name) const + { + if (CbFieldView Field = FindView(Name)) + { + return CbField(Field, *this); + } + return CbField(); + } + + /** Find a field by case-insensitive name comparison. */ + inline CbField FindIgnoreCase(std::string_view Name) const + { + if (CbFieldView Field = FindViewIgnoreCase(Name)) + { + return CbField(Field, *this); + } + return CbField(); + } + + /** Find a field by case-sensitive name comparison. */ + inline CbFieldView operator[](std::string_view Name) const { return Find(Name); } + + /** Access the object as an object field. */ + inline CbField AsField() const& { return CbField(CbObjectView::AsFieldView(), *this); } + + /** Access the object as an object field. */ + inline CbField AsField() && { return CbField(CbObjectView::AsFieldView(), std::move(*this)); } + +private: + friend inline CbFieldIterator begin(const CbObject& Object) { return Object.CreateIterator(); } + friend inline CbFieldIterator end(const CbObject&) { return CbFieldIterator(); } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +inline CbObject +CbField::AsObject() & +{ + return IsObject() ? CbObject(AsObjectView(), *this) : CbObject(); +} + +inline CbObject +CbField::AsObject() && +{ + return IsObject() ? CbObject(AsObjectView(), std::move(*this)) : CbObject(); +} + +inline CbArray +CbField::AsArray() & +{ + return IsArray() ? CbArray(AsArrayView(), *this) : CbArray(); +} + +inline CbArray +CbField::AsArray() && +{ + return IsArray() ? CbArray(AsArrayView(), std::move(*this)) : CbArray(); +} + +inline SharedBuffer +CbField::AsBinary(const SharedBuffer& Default) & +{ + const MemoryView View = AsBinaryView(); + return !HasError() ? SharedBuffer::MakeView(View, GetOuterBuffer()) : Default; +} + +inline SharedBuffer +CbField::AsBinary(const SharedBuffer& Default) && +{ + const MemoryView View = AsBinaryView(); + return !HasError() ? SharedBuffer::MakeView(View, std::move(*this).GetOuterBuffer()) : Default; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Load a compact binary field from an archive. + * + * The field may be an array or an object, which the caller can convert to by using AsArray or + * AsObject as appropriate. The buffer allocator is called to provide the buffer for the field + * to load into once its size has been determined. + * + * @param Ar Archive to read the field from. An error state is set on failure. + * @param Allocator Allocator for the buffer that the field is loaded into. + * @return A field with a reference to the allocated buffer, or a default field on failure. + */ +ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator); + +ZENCORE_API CbObject LoadCompactBinaryObject(IoBuffer&& Payload); +ZENCORE_API CbObject LoadCompactBinaryObject(const IoBuffer& Payload); +ZENCORE_API CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload); +ZENCORE_API CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload); + +/** + * Load a compact binary from JSON. + */ +ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json, std::string& Error); +ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Determine the size in bytes of the compact binary field at the start of the view. + * + * This may be called on an incomplete or invalid field, in which case the returned size is zero. + * A size can always be extracted from a valid field with no name if a view of at least the first + * 10 bytes is provided, regardless of field size. For fields with names, the size of view needed + * to calculate a size is at most 10 + MaxNameLen + MeasureVarUInt(MaxNameLen). + * + * This function can be used when streaming a field, for example, to determine the size of buffer + * to fill before attempting to construct a field from it. + * + * @param View A memory view that may contain the start of a field. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + */ +ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType); + +/** + * Try to determine the type and size of the compact binary field at the start of the view. + * + * This may be called on an incomplete or invalid field, in which case it will return false, with + * OutSize being 0 for invalid fields, otherwise the minimum view size necessary to make progress + * in measuring the field on the next call to this function. + * + * @note A return of true from this function does not indicate that the entire field is valid. + * + * @param InView A memory view that may contain the start of a field. + * @param OutType The type (with flags) of the field. None is written until a value is available. + * @param OutSize The total field size for a return of true, 0 for invalid fields, or the size to + * make progress in measuring the field on the next call to this function. + * @param InType HasFieldType means that InView contains the type. Otherwise, use the given type. + * @return true if the size of the field was determined, otherwise false. + */ +ZENCORE_API bool TryMeasureCompactBinary(MemoryView InView, + CbFieldType& OutType, + uint64_t& OutSize, + CbFieldType InType = CbFieldType::HasFieldType); + +inline CbFieldViewIterator +begin(CbFieldView& View) +{ + if (View.IsArray()) + { + return View.AsArrayView().CreateViewIterator(); + } + else if (View.IsObject()) + { + return View.AsObjectView().CreateViewIterator(); + } + + return CbFieldViewIterator(); +} + +inline CbFieldViewIterator +end(CbFieldView&) +{ + return CbFieldViewIterator(); +} + +void uson_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h new file mode 100644 index 000000000..4be8c2ba5 --- /dev/null +++ b/src/zencore/include/zencore/compactbinarybuilder.h @@ -0,0 +1,661 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> + +#include <zencore/enumflags.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/refcount.h> +#include <zencore/sha1.h> + +#include <atomic> +#include <memory> +#include <string> +#include <string_view> +#include <type_traits> +#include <vector> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +class CbAttachment; +class BinaryWriter; + +/** + * A writer for compact binary object, arrays, and fields. + * + * The writer produces a sequence of fields that can be saved to a provided memory buffer or into + * a new owned buffer. The typical use case is to write a single object, which can be accessed by + * calling Save().AsObjectRef() or Save(Buffer).AsObject(). + * + * The writer will assert on most incorrect usage and will always produce valid compact binary if + * provided with valid input. The writer does not check for invalid UTF-8 string encoding, object + * fields with duplicate names, or invalid compact binary being copied from another source. + * + * It is most convenient to use the streaming API for the writer, as demonstrated in the example. + * + * When writing a small amount of compact binary data, TCbWriter can be more efficient as it uses + * a fixed-size stack buffer for storage before spilling onto the heap. + * + * @see TCbWriter + * + * Example: + * + * CbObjectRef WriteObject() + * { + * CbWriter<256> Writer; + * Writer.BeginObject(); + * + * Writer << "Resize" << true; + * Writer << "MaxWidth" << 1024; + * Writer << "MaxHeight" << 1024; + * + * Writer.BeginArray(); + * Writer << "FormatA" << "FormatB" << "FormatC"; + * Writer.EndArray(); + * + * Writer.EndObject(); + * return Writer.Save().AsObjectRef(); + * } + */ +class CbWriter +{ +public: + ZENCORE_API CbWriter(); + ZENCORE_API ~CbWriter(); + + CbWriter(const CbWriter&) = delete; + CbWriter& operator=(const CbWriter&) = delete; + + /** Empty the writer without releasing any allocated memory. */ + ZENCORE_API void Reset(); + + /** + * Serialize the field(s) to an owned buffer and return it as an iterator. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + * The writer remains valid for further use when this function returns. + */ + ZENCORE_API CbFieldIterator Save(); + + /** + * Serialize the field(s) to memory. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + * The writer remains valid for further use when this function returns. + * + * @param Buffer A mutable memory view to write to. Must be exactly GetSaveSize() bytes. + * @return An iterator for the field(s) written to the buffer. + */ + ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer); + + ZENCORE_API void Save(BinaryWriter& Writer); + + /** + * The size of buffer (in bytes) required to serialize the fields that have been written. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + */ + ZENCORE_API uint64_t GetSaveSize() const; + + /** + * Sets the name of the next field to be written. + * + * It is not valid to call this function when writing a field inside an array. + * Names must be valid UTF-8 and must be unique within an object. + */ + ZENCORE_API CbWriter& SetName(std::string_view Name); + + /** Copy the value (not the name) of an existing field. */ + inline void AddField(std::string_view Name, const CbFieldView& Value) + { + SetName(Name); + AddField(Value); + } + + ZENCORE_API void AddField(const CbFieldView& Value); + + /** Copy the value (not the name) of an existing field. Holds a reference if owned. */ + inline void AddField(std::string_view Name, const CbField& Value) + { + SetName(Name); + AddField(Value); + } + ZENCORE_API void AddField(const CbField& Value); + + /** Begin a new object. Must have a matching call to EndObject. */ + inline void BeginObject(std::string_view Name) + { + SetName(Name); + BeginObject(); + } + ZENCORE_API void BeginObject(); + /** End an object after its fields have been written. */ + ZENCORE_API void EndObject(); + + /** Copy the value (not the name) of an existing object. */ + inline void AddObject(std::string_view Name, const CbObjectView& Value) + { + SetName(Name); + AddObject(Value); + } + ZENCORE_API void AddObject(const CbObjectView& Value); + /** Copy the value (not the name) of an existing object. Holds a reference if owned. */ + inline void AddObject(std::string_view Name, const CbObject& Value) + { + SetName(Name); + AddObject(Value); + } + ZENCORE_API void AddObject(const CbObject& Value); + + /** Begin a new array. Must have a matching call to EndArray. */ + inline void BeginArray(std::string_view Name) + { + SetName(Name); + BeginArray(); + } + ZENCORE_API void BeginArray(); + /** End an array after its fields have been written. */ + ZENCORE_API void EndArray(); + + /** Copy the value (not the name) of an existing array. */ + inline void AddArray(std::string_view Name, const CbArrayView& Value) + { + SetName(Name); + AddArray(Value); + } + ZENCORE_API void AddArray(const CbArrayView& Value); + /** Copy the value (not the name) of an existing array. Holds a reference if owned. */ + inline void AddArray(std::string_view Name, const CbArray& Value) + { + SetName(Name); + AddArray(Value); + } + ZENCORE_API void AddArray(const CbArray& Value); + + /** Write a null field. */ + inline void AddNull(std::string_view Name) + { + SetName(Name); + AddNull(); + } + ZENCORE_API void AddNull(); + + /** Write a binary field by copying Size bytes from Value. */ + inline void AddBinary(std::string_view Name, const void* Value, uint64_t Size) + { + SetName(Name); + AddBinary(Value, Size); + } + ZENCORE_API void AddBinary(const void* Value, uint64_t Size); + /** Write a binary field by copying the view. */ + inline void AddBinary(std::string_view Name, MemoryView Value) + { + SetName(Name); + AddBinary(Value); + } + inline void AddBinary(MemoryView Value) { AddBinary(Value.GetData(), Value.GetSize()); } + + /** Write a binary field by copying the buffer. Holds a reference if owned. */ + inline void AddBinary(std::string_view Name, IoBuffer Value) + { + SetName(Name); + AddBinary(std::move(Value)); + } + ZENCORE_API void AddBinary(IoBuffer Value); + ZENCORE_API void AddBinary(SharedBuffer Value); + + inline void AddBinary(std::string_view Name, const CompositeBuffer& Buffer) + { + SetName(Name); + AddBinary(Buffer); + } + ZENCORE_API void AddBinary(const CompositeBuffer& Buffer); + + /** Write a string field by copying the UTF-8 value. */ + inline void AddString(std::string_view Name, std::string_view Value) + { + SetName(Name); + AddString(Value); + } + ZENCORE_API void AddString(std::string_view Value); + /** Write a string field by converting the UTF-16 value to UTF-8. */ + inline void AddString(std::string_view Name, std::wstring_view Value) + { + SetName(Name); + AddString(Value); + } + ZENCORE_API void AddString(std::wstring_view Value); + + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, int32_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(int32_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, int64_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(int64_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, uint32_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(uint32_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, uint64_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(uint64_t Value); + + /** Write a float field from a 32-bit float value. */ + inline void AddFloat(std::string_view Name, float Value) + { + SetName(Name); + AddFloat(Value); + } + ZENCORE_API void AddFloat(float Value); + + /** Write a float field from a 64-bit float value. */ + inline void AddFloat(std::string_view Name, double Value) + { + SetName(Name); + AddFloat(Value); + } + ZENCORE_API void AddFloat(double Value); + + /** Write a bool field. */ + inline void AddBool(std::string_view Name, bool bValue) + { + SetName(Name); + AddBool(bValue); + } + ZENCORE_API void AddBool(bool bValue); + + /** Write a field referencing a compact binary attachment by its hash. */ + inline void AddObjectAttachment(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddObjectAttachment(Value); + } + ZENCORE_API void AddObjectAttachment(const IoHash& Value); + + /** Write a field referencing a binary attachment by its hash. */ + inline void AddBinaryAttachment(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddBinaryAttachment(Value); + } + ZENCORE_API void AddBinaryAttachment(const IoHash& Value); + + /** Write a field referencing the attachment by its hash. */ + inline void AddAttachment(std::string_view Name, const CbAttachment& Attachment) + { + SetName(Name); + AddAttachment(Attachment); + } + ZENCORE_API void AddAttachment(const CbAttachment& Attachment); + + /** Write a hash field. */ + inline void AddHash(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddHash(Value); + } + ZENCORE_API void AddHash(const IoHash& Value); + + /** Write a UUID field. */ + inline void AddUuid(std::string_view Name, const Guid& Value) + { + SetName(Name); + AddUuid(Value); + } + ZENCORE_API void AddUuid(const Guid& Value); + + /** Write an ObjectId field. */ + inline void AddObjectId(std::string_view Name, const Oid& Value) + { + SetName(Name); + AddObjectId(Value); + } + ZENCORE_API void AddObjectId(const Oid& Value); + + /** Write a date/time field with the specified count of 100ns ticks since the epoch. */ + inline void AddDateTimeTicks(std::string_view Name, int64_t Ticks) + { + SetName(Name); + AddDateTimeTicks(Ticks); + } + ZENCORE_API void AddDateTimeTicks(int64_t Ticks); + + /** Write a date/time field. */ + inline void AddDateTime(std::string_view Name, DateTime Value) + { + SetName(Name); + AddDateTime(Value); + } + ZENCORE_API void AddDateTime(DateTime Value); + + /** Write a time span field with the specified count of 100ns ticks. */ + inline void AddTimeSpanTicks(std::string_view Name, int64_t Ticks) + { + SetName(Name); + AddTimeSpanTicks(Ticks); + } + ZENCORE_API void AddTimeSpanTicks(int64_t Ticks); + + /** Write a time span field. */ + inline void AddTimeSpan(std::string_view Name, TimeSpan Value) + { + SetName(Name); + AddTimeSpan(Value); + } + ZENCORE_API void AddTimeSpan(TimeSpan Value); + + /** Private flags that are public to work with ENUM_CLASS_FLAGS. */ + enum class StateFlags : uint8_t; + +protected: + /** Reserve the specified size up front until the format is optimized. */ + ZENCORE_API explicit CbWriter(int64_t InitialSize); + +private: + friend CbWriter& operator<<(CbWriter& Writer, std::string_view NameOrValue); + + /** Begin writing a field. May be called twice for named fields. */ + void BeginField(); + + /** Finish writing a field by writing its type. */ + void EndField(CbFieldType Type); + + /** Set the field name if valid in this state, otherwise write add a string field. */ + ZENCORE_API void SetNameOrAddString(std::string_view NameOrValue); + + /** Returns a view of the name of the active field, if any, otherwise the empty view. */ + std::string_view GetActiveName() const; + + /** Remove field types after the first to make the sequence uniform. */ + void MakeFieldsUniform(int64_t FieldBeginOffset, int64_t FieldEndOffset); + + /** State of the object, array, or top-level field being written. */ + struct WriterState + { + StateFlags Flags{}; + /** The type of the fields in the sequence if uniform, otherwise None. */ + CbFieldType UniformType{}; + /** The offset of the start of the current field. */ + int64_t Offset{}; + /** The number of fields written in this state. */ + uint64_t Count{}; + }; + +private: + // This is a prototype-quality format for the writer. Using an array of bytes is inefficient, + // and will lead to many unnecessary copies and moves of the data to resize the array, insert + // object and array sizes, and remove field types for uniform objects and uniform arrays. The + // optimized format will be a list of power-of-two blocks and an optional first block that is + // provided externally, such as on the stack. That format will store the offsets that require + // object or array sizes to be inserted and field types to be removed, and will perform those + // operations only when saving to a buffer. + std::vector<uint8_t> Data; + std::vector<WriterState> States; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A writer for compact binary object, arrays, and fields that uses a fixed-size stack buffer. + * + * @see CbWriter + */ +template<uint32_t InlineBufferSize> +class FixedCbWriter : public CbWriter +{ +public: + inline FixedCbWriter() : CbWriter(InlineBufferSize) {} + + FixedCbWriter(const FixedCbWriter&) = delete; + FixedCbWriter& operator=(const FixedCbWriter&) = delete; + +private: + // Reserve the inline buffer now even though we are unable to use it. This will avoid causing + // new stack overflows when this functionality is properly implemented in the future. + uint8_t Buffer[InlineBufferSize]; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class CbObjectWriter : public CbWriter +{ +public: + CbObjectWriter() { BeginObject(); } + + ZENCORE_API CbObject Save() + { + Finalize(); + return CbWriter::Save().AsObject(); + } + + ZENCORE_API void Save(BinaryWriter& Writer) + { + Finalize(); + return CbWriter::Save(Writer); + } + + ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer) + { + ZEN_ASSERT(m_Finalized); + return CbWriter::Save(Buffer); + } + + uint64_t GetSaveSize() + { + ZEN_ASSERT(m_Finalized); + return CbWriter::GetSaveSize(); + } + + void Finalize() + { + if (m_Finalized == false) + { + EndObject(); + m_Finalized = true; + } + } + + CbObjectWriter(const CbWriter&) = delete; + CbObjectWriter& operator=(const CbWriter&) = delete; + +private: + bool m_Finalized = false; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Write the field name if valid in this state, otherwise write the string value. */ +inline CbWriter& +operator<<(CbWriter& Writer, std::string_view NameOrValue) +{ + Writer.SetNameOrAddString(NameOrValue); + return Writer; +} + +/** Write the field name if valid in this state, otherwise write the string value. */ +inline CbWriter& +operator<<(CbWriter& Writer, const char* NameOrValue) +{ + return Writer << std::string_view(NameOrValue); +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbFieldView& Value) +{ + Writer.AddField(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbField& Value) +{ + Writer.AddField(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbObjectView& Value) +{ + Writer.AddObject(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbObject& Value) +{ + Writer.AddObject(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbArrayView& Value) +{ + Writer.AddArray(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbArray& Value) +{ + Writer.AddArray(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, std::nullptr_t) +{ + Writer.AddNull(); + return Writer; +} + +#if defined(__clang__) && defined(__APPLE__) +/* Apple Clang has different types for uint64_t and size_t so an override is + needed here. Without it, Clang can't disambiguate integer overloads */ +inline CbWriter& +operator<<(CbWriter& Writer, std::size_t Value) +{ + Writer.AddInteger(uint64_t(Value)); + return Writer; +} +#endif + +inline CbWriter& +operator<<(CbWriter& Writer, std::wstring_view Value) +{ + Writer.AddString(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const wchar_t* Value) +{ + Writer.AddString(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, int32_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, int64_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, uint32_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, uint64_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, float Value) +{ + Writer.AddFloat(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, double Value) +{ + Writer.AddFloat(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, bool Value) +{ + Writer.AddBool(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbAttachment& Attachment) +{ + Writer.AddAttachment(Attachment); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const IoHash& Value) +{ + Writer.AddHash(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const Guid& Value) +{ + Writer.AddUuid(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const Oid& Value) +{ + Writer.AddObjectId(Value); + return Writer; +} + +ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value); +ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void usonbuilder_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h new file mode 100644 index 000000000..16f723edc --- /dev/null +++ b/src/zencore/include/zencore/compactbinarypackage.h @@ -0,0 +1,341 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> +#include <zencore/compress.h> +#include <zencore/iohash.h> + +#include <functional> +#include <span> +#include <variant> + +#ifdef GetObject +# error "windows.h pollution" +# undef GetObject +#endif + +namespace zen { + +class CbWriter; +class BinaryReader; +class BinaryWriter; +class IoBuffer; +class CbAttachment; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An attachment is either binary or compact binary and is identified by its hash. + * + * A compact binary attachment is also a valid binary attachment and may be accessed as binary. + * + * Attachments are serialized as one or two compact binary fields with no name. A Binary field is + * written first with its content. The content hash is omitted when the content size is zero, and + * is otherwise written as a BinaryReference or CompactBinaryReference depending on the type. + */ +class CbAttachment +{ +public: + /** Construct a null attachment. */ + CbAttachment() = default; + + /** Construct a compact binary attachment. Value is cloned if not owned. */ + inline explicit CbAttachment(const CbObject& InValue) : CbAttachment(InValue, nullptr) {} + + /** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */ + inline explicit CbAttachment(const CbObject& InValue, const IoHash& Hash) : CbAttachment(InValue, &Hash) {} + + /** Construct a raw binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue); + + /** Construct a raw binary attachment. Value is cloned if not owned. Hash must match Value. */ + ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue, const IoHash& Hash); + + /** Construct a raw binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(const CompositeBuffer& InValue); + + /** Construct a raw binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue); + + /** Construct a raw binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash); + + /** Construct a compressed binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash); + ZENCORE_API explicit CbAttachment(CompressedBuffer&& InValue, const IoHash& Hash); + + /** Reset this to a null attachment. */ + inline void Reset() { *this = CbAttachment(); } + + /** Whether the attachment has a value. */ + inline explicit operator bool() const { return !IsNull(); } + + /** Whether the attachment has a value. */ + ZENCORE_API [[nodiscard]] bool IsNull() const; + + /** Access the attachment as binary. Defaults to a null buffer on error. */ + ZENCORE_API [[nodiscard]] SharedBuffer AsBinary() const; + + /** Access the attachment as raw binary. Defaults to a null buffer on error. */ + ZENCORE_API [[nodiscard]] CompositeBuffer AsCompositeBinary() const; + + /** Access the attachment as compressed binary. Defaults to a null buffer if the attachment is null. */ + ZENCORE_API [[nodiscard]] CompressedBuffer AsCompressedBinary() const; + + /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ + ZENCORE_API [[nodiscard]] CbObject AsObject() const; + + /** Returns true if the attachment is binary */ + ZENCORE_API [[nodiscard]] bool IsBinary() const; + + /** Returns true if the attachment is compressed binary */ + ZENCORE_API [[nodiscard]] bool IsCompressedBinary() const; + + /** Returns whether the attachment is an object. */ + ZENCORE_API [[nodiscard]] bool IsObject() const; + + /** Returns the hash of the attachment value. */ + ZENCORE_API [[nodiscard]] IoHash GetHash() const; + + /** Compares attachments by their hash. Any discrepancy in type must be handled externally. */ + inline bool operator==(const CbAttachment& Attachment) const { return GetHash() == Attachment.GetHash(); } + inline bool operator!=(const CbAttachment& Attachment) const { return GetHash() != Attachment.GetHash(); } + inline bool operator<(const CbAttachment& Attachment) const { return GetHash() < Attachment.GetHash(); } + + /** + * Load the attachment from compact binary as written by Save. + * + * The attachment references the input iterator if it is owned, and otherwise clones the value. + * + * The iterator is advanced as attachment fields are consumed from it. + */ + ZENCORE_API bool TryLoad(CbFieldIterator& Fields); + + /** + * Load the attachment from compact binary as written by Save. + */ + ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc); + + /** + * Load the attachment from compact binary as written by Save. + */ + ZENCORE_API bool TryLoad(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc); + + /** Save the attachment into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(CbWriter& Writer) const; + + /** Save the attachment into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(BinaryWriter& Writer) const; + +private: + ZENCORE_API CbAttachment(const CbObject& Value, const IoHash* Hash); + + IoHash Hash; + std::variant<std::nullptr_t, CbObject, CompositeBuffer, CompressedBuffer> Value; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A package is a compact binary object with attachments for its external references. + * + * A package is basically a Merkle tree with compact binary as its root and other non-leaf nodes, + * and either binary or compact binary as its leaf nodes. A node references its child nodes using + * BinaryHash or FieldHash fields in its compact binary representation. + * + * It is invalid for a package to include attachments that are not referenced by its object or by + * one of its referenced compact binary attachments. When attachments are added explicitly, it is + * the responsibility of the package creator to follow this requirement. Attachments that are not + * referenced may not survive a round-trip through certain storage systems. + * + * It is valid for a package to exclude referenced attachments, but then it is the responsibility + * of the package consumer to have a mechanism for resolving those references when necessary. + * + * A package is serialized as a sequence of compact binary fields with no name. The object may be + * both preceded and followed by attachments. The object itself is written as an Object field and + * followed by its hash in a CompactBinaryReference field when the object is non-empty. A package + * ends with a Null field. The canonical order of components is the object and its hash, followed + * by the attachments ordered by hash, followed by a Null field. It is valid for the a package to + * have its components serialized in any order, provided there is at most one object and the null + * field is written last. + */ +class CbPackage +{ +public: + /** + * A function that resolves a hash to a buffer containing the data matching that hash. + * + * The resolver may return a null buffer to skip resolving an attachment for the hash. + */ + using AttachmentResolver = std::function<SharedBuffer(const IoHash& Hash)>; + + /** Construct a null package. */ + CbPackage() = default; + + /** + * Construct a package from a root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + */ + inline explicit CbPackage(CbObject InObject) { SetObject(std::move(InObject)); } + + /** + * Construct a package from a root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline explicit CbPackage(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), InResolver); } + + /** + * Construct a package from a root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + */ + inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), InObjectHash); } + + /** + * Construct a package from a root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver) + { + SetObject(std::move(InObject), InObjectHash, InResolver); + } + + /** Reset this to a null package. */ + inline void Reset() { *this = CbPackage(); } + + /** Whether the package has a non-empty object or attachments. */ + inline explicit operator bool() const { return !IsNull(); } + + /** Whether the package has an empty object and no attachments. */ + inline bool IsNull() const { return !Object && Attachments.size() == 0; } + + /** Returns the compact binary object for the package. */ + inline const CbObject& GetObject() const { return Object; } + + /** Returns the has of the compact binary object for the package. */ + inline const IoHash& GetObjectHash() const { return ObjectHash; } + + /** + * Set the root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + */ + inline void SetObject(CbObject InObject) { SetObject(std::move(InObject), nullptr, nullptr); } + + /** + * Set the root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline void SetObject(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), nullptr, &InResolver); } + + /** + * Set the root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + */ + inline void SetObject(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), &InObjectHash, nullptr); } + + /** + * Set the root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline void SetObject(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver) + { + SetObject(std::move(InObject), &InObjectHash, &InResolver); + } + + /** Returns the attachments in this package. */ + inline std::span<const CbAttachment> GetAttachments() const { return Attachments; } + + /** + * Find an attachment by its hash. + * + * @return The attachment, or null if the attachment is not found. + * @note The returned pointer is only valid until the attachments on this package are modified. + */ + ZENCORE_API const CbAttachment* FindAttachment(const IoHash& Hash) const; + + /** Find an attachment if it exists in the package. */ + inline const CbAttachment* FindAttachment(const CbAttachment& Attachment) const { return FindAttachment(Attachment.GetHash()); } + + /** Add the attachment to this package. */ + inline void AddAttachment(const CbAttachment& Attachment) { AddAttachment(Attachment, nullptr); } + + /** Add the attachment to this package, along with any references that can be resolved. */ + inline void AddAttachment(const CbAttachment& Attachment, AttachmentResolver Resolver) { AddAttachment(Attachment, &Resolver); } + + void AddAttachments(std::span<const CbAttachment> Attachments); + + /** + * Remove an attachment by hash. + * + * @return Number of attachments removed, which will be either 0 or 1. + */ + ZENCORE_API int32_t RemoveAttachment(const IoHash& Hash); + inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); } + + /** Compares packages by their object and attachment hashes. */ + ZENCORE_API bool Equals(const CbPackage& Package) const; + inline bool operator==(const CbPackage& Package) const { return Equals(Package); } + inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); } + + /** + * Load the object and attachments from compact binary as written by Save. + * + * The object and attachments reference the input iterator, if it is owned, and otherwise clones + * the object and attachments individually to make owned copies. + * + * The iterator is advanced as object and attachment fields are consumed from it. + */ + ZENCORE_API bool TryLoad(CbFieldIterator& Fields); + ZENCORE_API bool TryLoad(IoBuffer Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); + ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); + + /** Save the object and attachments into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(CbWriter& Writer) const; + + /** Save the object and attachments into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(BinaryWriter& Writer) const; + +private: + ZENCORE_API void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver); + ZENCORE_API void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver); + + void GatherAttachments(const CbObject& Object, AttachmentResolver Resolver); + + /** Attachments ordered by their hash. */ + std::vector<CbAttachment> Attachments; + CbObject Object; + IoHash ObjectHash; +}; + +namespace legacy { + void SaveCbAttachment(const CbAttachment& Attachment, CbWriter& Writer); + void SaveCbPackage(const CbPackage& Package, CbWriter& Writer); + void SaveCbPackage(const CbPackage& Package, BinaryWriter& Ar); + bool TryLoadCbPackage(CbPackage& Package, IoBuffer Buffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper = nullptr); + bool TryLoadCbPackage(CbPackage& Package, + BinaryReader& Reader, + BufferAllocator Allocator, + CbPackage::AttachmentResolver* Mapper = nullptr); +} // namespace legacy + +void usonpackage_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/compactbinaryvalidation.h b/src/zencore/include/zencore/compactbinaryvalidation.h new file mode 100644 index 000000000..b1fab9572 --- /dev/null +++ b/src/zencore/include/zencore/compactbinaryvalidation.h @@ -0,0 +1,197 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> +#include <zencore/enumflags.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/refcount.h> +#include <zencore/sha1.h> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +/** Flags for validating compact binary data. */ +enum class CbValidateMode : uint32_t +{ + /** Skip validation if no other validation modes are enabled. */ + None = 0, + + /** + * Validate that the value can be read and stays inside the bounds of the memory view. + * + * This is the minimum level of validation required to be able to safely read a field, array, + * or object without the risk of crashing or reading out of bounds. + */ + Default = 1 << 0, + + /** + * Validate that object fields have unique non-empty names and array fields have no names. + * + * Name validation failures typically do not inhibit reading the input, but duplicated fields + * cannot be looked up by name other than the first, and converting to other data formats can + * fail in the presence of naming issues. + */ + Names = 1 << 1, + + /** + * Validate that fields are serialized in the canonical format. + * + * Format validation failures typically do not inhibit reading the input. Values that fail in + * this mode require more memory than in the canonical format, and comparisons of such values + * for equality are not reliable. Examples of failures include uniform arrays or objects that + * were not encoded uniformly, variable-length integers that could be encoded in fewer bytes, + * or 64-bit floats that could be encoded in 32 bits without loss of precision. + */ + Format = 1 << 2, + + /** + * Validate that there is no padding after the value before the end of the memory view. + * + * Padding validation failures have no impact on the ability to read the input, but are using + * more memory than necessary. + */ + Padding = 1 << 3, + + /** + * Validate that a package or attachment has the expected fields. + */ + Package = 1 << 4, + + /** + * Validate that a package or attachment matches its saved hashes. + */ + PackageHash = 1 << 5, + + /** Perform all validation described above. */ + All = Default | Names | Format | Padding | Package | PackageHash, +}; + +ENUM_CLASS_FLAGS(CbValidateMode); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Flags for compact binary validation errors. Multiple flags may be combined. */ +enum class CbValidateError : uint32_t +{ + /** The input had no validation errors. */ + None = 0, + + // Mode: Default + + /** The input cannot be read without reading out of bounds. */ + OutOfBounds = 1 << 0, + /** The input has a field with an unrecognized or invalid type. */ + InvalidType = 1 << 1, + + // Mode: Names + + /** An object had more than one field with the same name. */ + DuplicateName = 1 << 2, + /** An object had a field with no name. */ + MissingName = 1 << 3, + /** An array field had a name. */ + ArrayName = 1 << 4, + + // Mode: Format + + /** A name or string payload is not valid UTF-8. */ + InvalidString = 1 << 5, + /** A size or integer payload can be encoded in fewer bytes. */ + InvalidInteger = 1 << 6, + /** A float64 payload can be encoded as a float32 without loss of precision. */ + InvalidFloat = 1 << 7, + /** An object has the same type for every field but is not uniform. */ + NonUniformObject = 1 << 8, + /** An array has the same type for every field and non-empty payloads but is not uniform. */ + NonUniformArray = 1 << 9, + + // Mode: Padding + + /** A value did not use the entire memory view given for validation. */ + Padding = 1 << 10, + + // Mode: Package + + /** The package or attachment had missing fields or fields out of order. */ + InvalidPackageFormat = 1 << 11, + /** The object or an attachment did not match the hash stored for it. */ + InvalidPackageHash = 1 << 12, + /** The package contained more than one copy of the same attachment. */ + DuplicateAttachments = 1 << 13, + /** The package contained more than one object. */ + MultiplePackageObjects = 1 << 14, + /** The package contained an object with no fields. */ + NullPackageObject = 1 << 15, + /** The package contained a null attachment. */ + NullPackageAttachment = 1 << 16, +}; + +ENUM_CLASS_FLAGS(CbValidateError); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Validate the compact binary data for one field in the view as specified by the mode flags. + * + * Only one top-level field is processed from the view, and validation recurses into any array or + * object within that field. To validate multiple consecutive top-level fields, call the function + * once for each top-level field. If the given view might contain multiple top-level fields, then + * either exclude the Padding flag from the Mode or use MeasureCompactBinary to break up the view + * into its constituent fields before validating. + * + * @param View A memory view containing at least one top-level field. + * @param Mode A combination of the flags for the types of validation to perform. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType); + +/** + * Validate the compact binary data for every field in the view as specified by the mode flags. + * + * This function expects the entire view to contain fields. Any trailing region of the view which + * does not contain a valid field will produce an OutOfBounds or InvalidType error instead of the + * Padding error that would be produced by the single field validation function. + * + * @see ValidateCompactBinary + */ +ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode); + +/** + * Validate the compact binary attachment pointed to by the view as specified by the mode flags. + * + * The attachment is validated with ValidateCompactBinary by using the validation mode specified. + * Include ECbValidateMode::Package to validate the attachment format and hash. + * + * @see ValidateCompactBinary + * + * @param View A memory view containing a package. + * @param Mode A combination of the flags for the types of validation to perform. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateObjectAttachment(MemoryView View, CbValidateMode Mode); + +/** + * Validate the compact binary package pointed to by the view as specified by the mode flags. + * + * The package, and attachments, are validated with ValidateCompactBinary by using the validation + * mode specified. Include ECbValidateMode::Package to validate the package format and hashes. + * + * @see ValidateCompactBinary + * + * @param View A memory view containing a package. + * @param Mode A combination of the flags for the types of validation to perform. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void usonvalidation_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/compactbinaryvalue.h b/src/zencore/include/zencore/compactbinaryvalue.h new file mode 100644 index 000000000..0124a8983 --- /dev/null +++ b/src/zencore/include/zencore/compactbinaryvalue.h @@ -0,0 +1,290 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/compactbinary.h> +#include <zencore/endian.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/memory.h> + +namespace zen { + +namespace CompactBinaryPrivate { + + template<typename T> + static constexpr inline T ReadUnaligned(const void* const Memory) + { +#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS + return *static_cast<const T*>(Memory); +#else + T Value; + memcpy(&Value, Memory, sizeof(Value)); + return Value; +#endif + } +} // namespace CompactBinaryPrivate +/** + * A type that provides unchecked access to compact binary values. + * + * The main purpose of the type is to efficiently switch on field type. For every other use case, + * prefer to use the field, array, and object types directly. The accessors here do not check the + * type before reading the value, which means they can read out of bounds even on a valid compact + * binary value if the wrong accessor is used. + */ +class CbValue +{ +public: + CbValue(CbFieldType Type, const void* Value); + + CbObjectView AsObjectView() const; + CbArrayView AsArrayView() const; + + MemoryView AsBinary() const; + + /** Access as a string. Checks for range errors and uses the default if OutError is not null. */ + std::string_view AsString(CbFieldError* OutError = nullptr, std::string_view Default = std::string_view()) const; + + /** Access as a string as UTF8. Checks for range errors and uses the default if OutError is not null. */ + std::u8string_view AsU8String(CbFieldError* OutError = nullptr, std::u8string_view Default = std::u8string_view()) const; + + /** + * Access as an integer, with both positive and negative values returned as unsigned. + * + * Checks for range errors and uses the default if OutError is not null. + */ + uint64_t AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError = nullptr, uint64_t Default = 0) const; + + uint64_t AsIntegerPositive() const; + int64_t AsIntegerNegative() const; + + float AsFloat32() const; + double AsFloat64() const; + + bool AsBool() const; + + inline IoHash AsObjectAttachment() const { return AsHash(); } + inline IoHash AsBinaryAttachment() const { return AsHash(); } + inline IoHash AsAttachment() const { return AsHash(); } + + IoHash AsHash() const; + Guid AsUuid() const; + + int64_t AsDateTimeTicks() const; + int64_t AsTimeSpanTicks() const; + + Oid AsObjectId() const; + + CbCustomById AsCustomById() const; + CbCustomByName AsCustomByName() const; + + inline CbFieldType GetType() const { return Type; } + inline const void* GetData() const { return Data; } + +private: + const void* Data; + CbFieldType Type; +}; + +inline CbFieldView::CbFieldView(const CbValue& InValue) : Type(InValue.GetType()), Payload(InValue.GetData()) +{ +} + +inline CbValue +CbFieldView::GetValue() const +{ + return CbValue(CbFieldTypeOps::GetType(Type), Payload); +} + +inline CbValue::CbValue(CbFieldType InType, const void* InValue) : Data(InValue), Type(InType) +{ +} + +inline CbObjectView +CbValue::AsObjectView() const +{ + return CbObjectView(*this); +} + +inline CbArrayView +CbValue::AsArrayView() const +{ + return CbArrayView(*this); +} + +inline MemoryView +CbValue::AsBinary() const +{ + const uint8_t* const Bytes = static_cast<const uint8_t*>(Data); + uint32_t ValueSizeByteCount; + const uint64_t ValueSize = ReadVarUInt(Bytes, ValueSizeByteCount); + return MakeMemoryView(Bytes + ValueSizeByteCount, ValueSize); +} + +inline std::string_view +CbValue::AsString(CbFieldError* OutError, std::string_view Default) const +{ + const char* const Chars = static_cast<const char*>(Data); + uint32_t ValueSizeByteCount; + const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); + + if (OutError) + { + if (ValueSize >= (uint64_t(1) << 31)) + { + *OutError = CbFieldError::RangeError; + return Default; + } + *OutError = CbFieldError::None; + } + + return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); +} + +inline std::u8string_view +CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const +{ + const char8_t* const Chars = static_cast<const char8_t*>(Data); + uint32_t ValueSizeByteCount; + const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); + + if (OutError) + { + if (ValueSize >= (uint64_t(1) << 31)) + { + *OutError = CbFieldError::RangeError; + return Default; + } + *OutError = CbFieldError::None; + } + + return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); +} + +inline uint64_t +CbValue::AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError, uint64_t Default) const +{ + // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero. + const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1); + const uint64_t IsNegative = uint8_t(Type) & 1; + + uint32_t MagnitudeByteCount; + const uint64_t Magnitude = ReadVarUInt(Data, MagnitudeByteCount); + const uint64_t Value = Magnitude ^ -int64_t(IsNegative); + + if (OutError) + { + const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned); + *OutError = IsInRange ? CbFieldError::None : CbFieldError::RangeError; + + const uint64_t UseValueMask = -int64_t(IsInRange); + return (Value & UseValueMask) | (Default & ~UseValueMask); + } + + return Value; +} + +inline uint64_t +CbValue::AsIntegerPositive() const +{ + uint32_t MagnitudeByteCount; + return ReadVarUInt(Data, MagnitudeByteCount); +} + +inline int64_t +CbValue::AsIntegerNegative() const +{ + uint32_t MagnitudeByteCount; + return int64_t(ReadVarUInt(Data, MagnitudeByteCount)) ^ -int64_t(1); +} + +inline float +CbValue::AsFloat32() const +{ + const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint32_t>(Data)); + return reinterpret_cast<const float&>(Value); +} + +inline double +CbValue::AsFloat64() const +{ + const uint64_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint64_t>(Data)); + return reinterpret_cast<const double&>(Value); +} + +inline bool +CbValue::AsBool() const +{ + return uint8_t(Type) & 1; +} + +inline IoHash +CbValue::AsHash() const +{ + return IoHash::MakeFrom(Data); +} + +inline Guid +CbValue::AsUuid() const +{ + Guid Value; + memcpy(&Value, Data, sizeof(Guid)); + Value.A = FromNetworkOrder(Value.A); + Value.B = FromNetworkOrder(Value.B); + Value.C = FromNetworkOrder(Value.C); + Value.D = FromNetworkOrder(Value.D); + return Value; +} + +inline int64_t +CbValue::AsDateTimeTicks() const +{ + return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data)); +} + +inline int64_t +CbValue::AsTimeSpanTicks() const +{ + return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data)); +} + +inline Oid +CbValue::AsObjectId() const +{ + return Oid::FromMemory(Data); +} + +inline CbCustomById +CbValue::AsCustomById() const +{ + const uint8_t* Bytes = static_cast<const uint8_t*>(Data); + uint32_t DataSizeByteCount; + const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount); + Bytes += DataSizeByteCount; + + CbCustomById Value; + uint32_t TypeIdByteCount; + Value.Id = ReadVarUInt(Bytes, TypeIdByteCount); + Value.Data = MakeMemoryView(Bytes + TypeIdByteCount, DataSize - TypeIdByteCount); + return Value; +} + +inline CbCustomByName +CbValue::AsCustomByName() const +{ + const uint8_t* Bytes = static_cast<const uint8_t*>(Data); + uint32_t DataSizeByteCount; + const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount); + Bytes += DataSizeByteCount; + + uint32_t TypeNameLenByteCount; + const uint64_t TypeNameLen = ReadVarUInt(Bytes, TypeNameLenByteCount); + Bytes += TypeNameLenByteCount; + + CbCustomByName Value; + Value.Name = std::u8string_view(reinterpret_cast<const char8_t*>(Bytes), static_cast<std::u8string_view::size_type>(TypeNameLen)); + Value.Data = MakeMemoryView(Bytes + TypeNameLen, DataSize - TypeNameLen - TypeNameLenByteCount); + return Value; +} + +} // namespace zen diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h new file mode 100644 index 000000000..4e4b4d002 --- /dev/null +++ b/src/zencore/include/zencore/compositebuffer.h @@ -0,0 +1,142 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/sharedbuffer.h> +#include <zencore/zencore.h> + +#include <functional> +#include <span> +#include <vector> + +namespace zen { + +/** + * CompositeBuffer is a non-contiguous buffer composed of zero or more immutable shared buffers. + * + * A composite buffer is most efficient when its segments are consumed as they are, but it can be + * flattened into a contiguous buffer, when necessary, by calling Flatten(). Ownership of segment + * buffers is not changed on construction, but if ownership of segments is required then that can + * be guaranteed by calling MakeOwned(). + */ + +class CompositeBuffer +{ +public: + /** + * Construct a composite buffer by concatenating the buffers. Does not enforce ownership. + * + * Buffer parameters may be SharedBuffer, CompositeBuffer, or std::vector<SharedBuffer>. + */ + template<typename... BufferTypes> + inline explicit CompositeBuffer(BufferTypes&&... Buffers) + { + if constexpr (sizeof...(Buffers) > 0) + { + m_Segments.reserve((GetBufferCount(std::forward<BufferTypes>(Buffers)) + ...)); + (AppendBuffers(std::forward<BufferTypes>(Buffers)), ...); + std::erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); }); + } + } + + /** Reset this to null. */ + ZENCORE_API void Reset(); + + /** Returns the total size of the composite buffer in bytes. */ + [[nodiscard]] ZENCORE_API uint64_t GetSize() const; + + /** Returns the segments that the buffer is composed from. */ + [[nodiscard]] inline std::span<const SharedBuffer> GetSegments() const { return std::span<const SharedBuffer>{m_Segments}; } + + /** Returns true if the composite buffer is not null. */ + [[nodiscard]] inline explicit operator bool() const { return !IsNull(); } + + /** Returns true if the composite buffer is null. */ + [[nodiscard]] inline bool IsNull() const { return m_Segments.empty(); } + + /** Returns true if every segment in the composite buffer is owned. */ + [[nodiscard]] ZENCORE_API bool IsOwned() const; + + /** Returns a copy of the buffer where every segment is owned. */ + [[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() const&; + [[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() &&; + + /** Returns the concatenation of the segments into a contiguous buffer. */ + [[nodiscard]] ZENCORE_API SharedBuffer Flatten() const&; + [[nodiscard]] ZENCORE_API SharedBuffer Flatten() &&; + + /** Returns the middle part of the buffer by taking the size starting at the offset. */ + [[nodiscard]] ZENCORE_API CompositeBuffer Mid(uint64_t Offset, uint64_t Size = ~uint64_t(0)) const; + + /** + * Returns a view of the range if contained by one segment, otherwise a view of a copy of the range. + * + * @note CopyBuffer is reused if large enough, and otherwise allocated when needed. + * + * @param Offset The byte offset in this buffer that the range starts at. + * @param Size The number of bytes in the range to view or copy. + * @param CopyBuffer The buffer to write the copy into if a copy is required. + */ + [[nodiscard]] ZENCORE_API MemoryView ViewOrCopyRange(uint64_t Offset, uint64_t Size, UniqueBuffer& CopyBuffer) const; + + /** + * Copies a range of the buffer to a contiguous region of memory. + * + * @param Target The view to copy to. Must be no larger than the data available at the offset. + * @param Offset The byte offset in this buffer to start copying from. + */ + ZENCORE_API void CopyTo(MutableMemoryView Target, uint64_t Offset = 0) const; + + /** + * Invokes a visitor with a view of each segment that intersects with a range. + * + * @param Offset The byte offset in this buffer to start visiting from. + * @param Size The number of bytes in the range to visit. + * @param Visitor The visitor to invoke from zero to GetSegments().Num() times. + */ + ZENCORE_API void IterateRange(uint64_t Offset, uint64_t Size, std::function<void(MemoryView View)> Visitor) const; + ZENCORE_API void IterateRange(uint64_t Offset, + uint64_t Size, + std::function<void(MemoryView View, const SharedBuffer& ViewOuter)> Visitor) const; + + struct Iterator + { + size_t SegmentIndex = 0; + uint64_t OffsetInSegment = 0; + }; + ZENCORE_API Iterator GetIterator(uint64_t Offset) const; + ZENCORE_API MemoryView ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const; + ZENCORE_API void CopyTo(MutableMemoryView Target, Iterator& It) const; + + /** A null composite buffer. */ + static const CompositeBuffer Null; + +private: + static inline size_t GetBufferCount(const CompositeBuffer& Buffer) { return Buffer.m_Segments.size(); } + inline void AppendBuffers(const CompositeBuffer& Buffer) + { + m_Segments.insert(m_Segments.end(), begin(Buffer.m_Segments), end(Buffer.m_Segments)); + } + inline void AppendBuffers(CompositeBuffer&& Buffer) + { + // TODO: this operates just like the by-reference version above + m_Segments.insert(m_Segments.end(), begin(Buffer.m_Segments), end(Buffer.m_Segments)); + } + + static inline size_t GetBufferCount(const SharedBuffer&) { return 1; } + inline void AppendBuffers(const SharedBuffer& Buffer) { m_Segments.push_back(Buffer); } + inline void AppendBuffers(SharedBuffer&& Buffer) { m_Segments.push_back(std::move(Buffer)); } + + static inline size_t GetBufferCount(std::vector<SharedBuffer>&& Container) { return Container.size(); } + inline void AppendBuffers(std::vector<SharedBuffer>&& Container) + { + m_Segments.insert(m_Segments.end(), begin(Container), end(Container)); + } + +private: + std::vector<SharedBuffer> m_Segments; +}; + +void compositebuffer_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h new file mode 100644 index 000000000..99ce20d8a --- /dev/null +++ b/src/zencore/include/zencore/compress.h @@ -0,0 +1,165 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore/zencore.h" + +#include "zencore/blake3.h" +#include "zencore/compositebuffer.h" + +namespace zen { + +enum class OodleCompressor : uint8_t +{ + NotSet = 0, + Selkie = 1, + Mermaid = 2, + Kraken = 3, + Leviathan = 4, +}; + +enum class OodleCompressionLevel : int8_t +{ + HyperFast4 = -4, + HyperFast3 = -3, + HyperFast2 = -2, + HyperFast1 = -1, + None = 0, + SuperFast = 1, + VeryFast = 2, + Fast = 3, + Normal = 4, + Optimal1 = 5, + Optimal2 = 6, + Optimal3 = 7, + Optimal4 = 8, +}; + +/** + * A compressed buffer stores compressed data in a self-contained format. + * + * A buffer is self-contained in the sense that it can be decompressed without external knowledge + * of the compression format or the size of the raw data. + */ +class CompressedBuffer +{ +public: + /** + * Compress the buffer using the specified compressor and compression level. + * + * Data that does not compress will be return uncompressed, as if with level None. + * + * @note Using a level of None will return a buffer that references owned raw data. + * + * @param RawData The raw data to be compressed. + * @param Compressor The compressor to encode with. May use NotSet if level is None. + * @param CompressionLevel The compression level to encode with. + * @param BlockSize The power-of-two block size to encode raw data in. 0 is default. + * @return An owned compressed buffer, or null on error. + */ + [[nodiscard]] ZENCORE_API static CompressedBuffer Compress(const CompositeBuffer& RawData, + OodleCompressor Compressor = OodleCompressor::Mermaid, + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0); + [[nodiscard]] ZENCORE_API static CompressedBuffer Compress(const SharedBuffer& RawData, + OodleCompressor Compressor = OodleCompressor::Mermaid, + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0); + + /** + * Construct from a compressed buffer previously created by Compress(). + * + * @return A compressed buffer, or null on error, such as an invalid format or corrupt header. + */ + [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData, + IoHash& OutRawHash, + uint64_t& OutRawSize); + [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData, + IoHash& OutRawHash, + uint64_t& OutRawSize); + [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData, + IoHash& OutRawHash, + uint64_t& OutRawSize); + [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData, + IoHash& OutRawHash, + uint64_t& OutRawSize); + [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData); + [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData); + [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize); + [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(const IoBuffer& CompressedData, + IoHash& OutRawHash, + uint64_t& OutRawSize); + + /** Reset this to null. */ + inline void Reset() { CompressedData.Reset(); } + + /** Returns true if the compressed buffer is not null. */ + [[nodiscard]] inline explicit operator bool() const { return !IsNull(); } + + /** Returns true if the compressed buffer is null. */ + [[nodiscard]] inline bool IsNull() const { return CompressedData.IsNull(); } + + /** Returns true if the composite buffer is owned. */ + [[nodiscard]] inline bool IsOwned() const { return CompressedData.IsOwned(); } + + /** Returns a copy of the compressed buffer that owns its underlying memory. */ + [[nodiscard]] inline CompressedBuffer MakeOwned() const& { return FromCompressedNoValidate(CompressedData.MakeOwned()); } + [[nodiscard]] inline CompressedBuffer MakeOwned() && { return FromCompressedNoValidate(std::move(CompressedData).MakeOwned()); } + + /** Returns a composite buffer containing the compressed data. May be null. May not be owned. */ + [[nodiscard]] inline const CompositeBuffer& GetCompressed() const& { return CompressedData; } + [[nodiscard]] inline CompositeBuffer GetCompressed() && { return std::move(CompressedData); } + + /** Returns the size of the compressed data. Zero if this is null. */ + [[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); } + + /** Returns the size of the raw data. Zero on error or if this is empty or null. */ + [[nodiscard]] ZENCORE_API uint64_t DecodeRawSize() const; + + /** Returns the hash of the raw data. Zero on error or if this is null. */ + [[nodiscard]] ZENCORE_API IoHash DecodeRawHash() const; + + [[nodiscard]] ZENCORE_API CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const; + + /** + * Returns the compressor and compression level used by this buffer. + * + * The compressor and compression level may differ from those specified when creating the buffer + * because an incompressible buffer is stored with no compression. Parameters cannot be accessed + * if this is null or uses a method other than Oodle, in which case this returns false. + * + * @return True if parameters were written, otherwise false. + */ + [[nodiscard]] ZENCORE_API bool TryGetCompressParameters(OodleCompressor& OutCompressor, + OodleCompressionLevel& OutCompressionLevel, + uint64_t& OutBlockSize) const; + + /** + * Decompress into a memory view that is less or equal GetRawSize() bytes. + */ + [[nodiscard]] ZENCORE_API bool TryDecompressTo(MutableMemoryView RawView, uint64_t RawOffset = 0) const; + + /** + * Decompress into an owned buffer. + * + * @return An owned buffer containing the raw data, or null on error. + */ + [[nodiscard]] ZENCORE_API SharedBuffer Decompress(uint64_t RawOffset = 0, uint64_t RawSize = ~uint64_t(0)) const; + + /** + * Decompress into an owned composite buffer. + * + * @return An owned buffer containing the raw data, or null on error. + */ + [[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const; + + /** A null compressed buffer. */ + static const CompressedBuffer Null; + +private: + CompositeBuffer CompressedData; +}; + +void compress_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/config.h.in b/src/zencore/include/zencore/config.h.in new file mode 100644 index 000000000..3372eca2a --- /dev/null +++ b/src/zencore/include/zencore/config.h.in @@ -0,0 +1,16 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +// NOTE: Generated from config.h.in + +#define ZEN_CFG_VERSION "${VERSION}" +#define ZEN_CFG_VERSION_MAJOR ${VERSION_MAJOR} +#define ZEN_CFG_VERSION_MINOR ${VERSION_MINOR} +#define ZEN_CFG_VERSION_ALTER ${VERSION_ALTER} +#define ZEN_CFG_VERSION_BUILD ${VERSION_BUILD} +#define ZEN_CFG_VERSION_BRANCH "${GIT_BRANCH}" +#define ZEN_CFG_VERSION_COMMIT "${GIT_COMMIT}" +#define ZEN_CFG_VERSION_BUILD_STRING "${VERSION}-${plat}-${arch}-${mode}" +#define ZEN_CFG_VERSION_BUILD_STRING_FULL "${VERSION}-${VERSION_BUILD}-${plat}-${arch}-${mode}-${GIT_COMMIT}" +#define ZEN_CFG_SCHEMA_VERSION ${ZEN_SCHEMA_VERSION} diff --git a/src/zencore/include/zencore/crc32.h b/src/zencore/include/zencore/crc32.h new file mode 100644 index 000000000..336bda77e --- /dev/null +++ b/src/zencore/include/zencore/crc32.h @@ -0,0 +1,13 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +namespace zen { + +uint32_t MemCrc32(const void* InData, size_t Length, uint32_t Crc = 0); +uint32_t MemCrc32_Deprecated(const void* InData, size_t Length, uint32_t Crc = 0); +uint32_t StrCrc_Deprecated(const char* Data); + +} // namespace zen diff --git a/src/zencore/include/zencore/crypto.h b/src/zencore/include/zencore/crypto.h new file mode 100644 index 000000000..83d416b0f --- /dev/null +++ b/src/zencore/include/zencore/crypto.h @@ -0,0 +1,77 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/memory.h> +#include <zencore/zencore.h> + +#include <memory> +#include <optional> + +namespace zen { + +template<size_t BitCount> +struct CryptoBits +{ +public: + static constexpr size_t ByteCount = BitCount / 8; + + CryptoBits() = default; + + bool IsNull() const { return memcmp(&m_Bits, &Zero, ByteCount) == 0; } + bool IsValid() const { return IsNull() == false; } + + size_t GetSize() const { return ByteCount; } + size_t GetBitCount() const { return BitCount; } + + MemoryView GetView() const { return MemoryView(m_Bits, ByteCount); } + + static CryptoBits FromMemoryView(MemoryView Bits) + { + if (Bits.GetSize() != ByteCount) + { + return CryptoBits(); + } + + return CryptoBits(Bits); + } + + static CryptoBits FromString(std::string_view Str) { return FromMemoryView(MakeMemoryView(Str)); } + +private: + CryptoBits(MemoryView Bits) + { + ZEN_ASSERT(Bits.GetSize() == GetSize()); + memcpy(&m_Bits, Bits.GetData(), GetSize()); + } + + static constexpr uint8_t Zero[ByteCount] = {0}; + + uint8_t m_Bits[ByteCount] = {0}; +}; + +using AesKey256Bit = CryptoBits<256>; +using AesIV128Bit = CryptoBits<128>; + +class Aes +{ +public: + static constexpr size_t BlockSize = 16; + + static MemoryView Encrypt(const AesKey256Bit& Key, + const AesIV128Bit& IV, + MemoryView In, + MutableMemoryView Out, + std::optional<std::string>& Reason); + + static MemoryView Decrypt(const AesKey256Bit& Key, + const AesIV128Bit& IV, + MemoryView In, + MutableMemoryView Out, + std::optional<std::string>& Reason); +}; + +void crypto_forcelink(); + +} // namespace zen diff --git a/src/zencore/include/zencore/endian.h b/src/zencore/include/zencore/endian.h new file mode 100644 index 000000000..7a9e6b44c --- /dev/null +++ b/src/zencore/include/zencore/endian.h @@ -0,0 +1,113 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <cstdint> + +namespace zen { + +inline uint16_t +ByteSwap(uint16_t x) +{ +#if ZEN_COMPILER_MSC + return _byteswap_ushort(x); +#else + return __builtin_bswap16(x); +#endif +} + +inline uint32_t +ByteSwap(uint32_t x) +{ +#if ZEN_COMPILER_MSC + return _byteswap_ulong(x); +#else + return __builtin_bswap32(x); +#endif +} + +inline uint64_t +ByteSwap(uint64_t x) +{ +#if ZEN_COMPILER_MSC + return _byteswap_uint64(x); +#else + return __builtin_bswap64(x); +#endif +} + +inline uint16_t +FromNetworkOrder(uint16_t x) +{ + return ByteSwap(x); +} + +inline uint32_t +FromNetworkOrder(uint32_t x) +{ + return ByteSwap(x); +} + +inline uint64_t +FromNetworkOrder(uint64_t x) +{ + return ByteSwap(x); +} + +inline uint16_t +FromNetworkOrder(int16_t x) +{ + return ByteSwap(uint16_t(x)); +} + +inline uint32_t +FromNetworkOrder(int32_t x) +{ + return ByteSwap(uint32_t(x)); +} + +inline uint64_t +FromNetworkOrder(int64_t x) +{ + return ByteSwap(uint64_t(x)); +} + +inline uint16_t +ToNetworkOrder(uint16_t x) +{ + return ByteSwap(x); +} + +inline uint32_t +ToNetworkOrder(uint32_t x) +{ + return ByteSwap(x); +} + +inline uint64_t +ToNetworkOrder(uint64_t x) +{ + return ByteSwap(x); +} + +inline uint16_t +ToNetworkOrder(int16_t x) +{ + return ByteSwap(uint16_t(x)); +} + +inline uint32_t +ToNetworkOrder(int32_t x) +{ + return ByteSwap(uint32_t(x)); +} + +inline uint64_t +ToNetworkOrder(int64_t x) +{ + return ByteSwap(uint64_t(x)); +} + +} // namespace zen diff --git a/src/zencore/include/zencore/enumflags.h b/src/zencore/include/zencore/enumflags.h new file mode 100644 index 000000000..ebe747bf0 --- /dev/null +++ b/src/zencore/include/zencore/enumflags.h @@ -0,0 +1,61 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +namespace zen { + +// Enum class helpers + +// Defines all bitwise operators for enum classes so it can be (mostly) used as a regular flags enum +#define ENUM_CLASS_FLAGS(Enum) \ + inline Enum& operator|=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \ + inline Enum& operator&=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \ + inline Enum& operator^=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator|(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator&(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator^(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \ + inline constexpr bool operator!(Enum E) { return !(__underlying_type(Enum))E; } \ + inline constexpr Enum operator~(Enum E) { return (Enum) ~(__underlying_type(Enum))E; } + +// Friends all bitwise operators for enum classes so the definition can be kept private / protected. +#define FRIEND_ENUM_CLASS_FLAGS(Enum) \ + friend Enum& operator|=(Enum& Lhs, Enum Rhs); \ + friend Enum& operator&=(Enum& Lhs, Enum Rhs); \ + friend Enum& operator^=(Enum& Lhs, Enum Rhs); \ + friend constexpr Enum operator|(Enum Lhs, Enum Rhs); \ + friend constexpr Enum operator&(Enum Lhs, Enum Rhs); \ + friend constexpr Enum operator^(Enum Lhs, Enum Rhs); \ + friend constexpr bool operator!(Enum E); \ + friend constexpr Enum operator~(Enum E); + +template<typename Enum> +constexpr bool +EnumHasAllFlags(Enum Flags, Enum Contains) +{ + return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) == ((__underlying_type(Enum))Contains); +} + +template<typename Enum> +constexpr bool +EnumHasAnyFlags(Enum Flags, Enum Contains) +{ + return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) != 0; +} + +template<typename Enum> +void +EnumAddFlags(Enum& Flags, Enum FlagsToAdd) +{ + Flags |= FlagsToAdd; +} + +template<typename Enum> +void +EnumRemoveFlags(Enum& Flags, Enum FlagsToRemove) +{ + Flags &= ~FlagsToRemove; +} + +} // namespace zen diff --git a/src/zencore/include/zencore/except.h b/src/zencore/include/zencore/except.h new file mode 100644 index 000000000..c61db5ba9 --- /dev/null +++ b/src/zencore/include/zencore/except.h @@ -0,0 +1,57 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/string.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#else +# include <errno.h> +#endif +#if __has_include("source_location") +# include <source_location> +#endif +#include <string> +#include <system_error> + +namespace zen { + +#if ZEN_PLATFORM_WINDOWS +ZENCORE_API void ThrowSystemException [[noreturn]] (HRESULT hRes, std::string_view Message); +#endif // ZEN_PLATFORM_WINDOWS + +#if defined(__cpp_lib_source_location) +ZENCORE_API void ThrowLastErrorImpl [[noreturn]] (std::string_view Message, const std::source_location& Location); +# define ThrowLastError(Message) ThrowLastErrorImpl(Message, std::source_location::current()) +#else +ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message); +#endif + +ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); + +ZENCORE_API std::string GetLastErrorAsString(); +ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode); + +inline int32_t +GetLastError() +{ +#if ZEN_PLATFORM_WINDOWS + return ::GetLastError(); +#else + return errno; +#endif +} + +inline std::error_code +MakeErrorCode(uint32_t ErrorCode) noexcept +{ + return std::error_code(ErrorCode, std::system_category()); +} + +inline std::error_code +MakeErrorCodeFromLastError() noexcept +{ + return std::error_code(zen::GetLastError(), std::system_category()); +} + +} // namespace zen diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h new file mode 100644 index 000000000..fa5f94170 --- /dev/null +++ b/src/zencore/include/zencore/filesystem.h @@ -0,0 +1,190 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/iobuffer.h> +#include <zencore/string.h> + +#include <filesystem> +#include <functional> + +namespace zen { + +class IoBuffer; + +/** Delete directory (after deleting any contents) + */ +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); + +/** Ensure directory exists. + + Will also create any required parent directories + */ +ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); + +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); + +/** Map native file handle to a path + */ +ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle); + +ZENCORE_API std::filesystem::path GetRunningExecutablePath(); + +/** Set the max open file handle count to max allowed for the current process on Linux and MacOS + */ +ZENCORE_API void MaximizeOpenFileCount(); + +struct FileContents +{ + std::vector<IoBuffer> Data; + std::error_code ErrorCode; + + IoBuffer Flatten(); +}; + +ZENCORE_API FileContents ReadStdIn(); +ZENCORE_API FileContents ReadFile(std::filesystem::path Path); +ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc); +ZENCORE_API void WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount); +ZENCORE_API void WriteFile(std::filesystem::path Path, IoBuffer Data); + +struct CopyFileOptions +{ + bool EnableClone = true; + bool MustClone = false; +}; + +ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); +ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); + +ZENCORE_API void PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out); +ZENCORE_API std::string PathToUtf8(const std::filesystem::path& Path); + +extern template class StringBuilderImpl<std::filesystem::path::value_type>; + +/** + * Helper class for building paths. Backed by a string builder. + * + */ +class PathBuilderBase : public StringBuilderImpl<std::filesystem::path::value_type> +{ +private: + using Super = StringBuilderImpl<std::filesystem::path::value_type>; + +protected: + using CharType = std::filesystem::path::value_type; + using ViewType = std::basic_string_view<CharType>; + +public: + void Append(const std::filesystem::path& Rhs) { Super::Append(Rhs.c_str()); } + void operator/=(const std::filesystem::path& Rhs) { this->operator/=(Rhs.c_str()); }; + void operator/=(const CharType* Rhs) + { + AppendSeparator(); + Super::Append(Rhs); + } + operator ViewType() const { return ToView(); } + std::basic_string_view<CharType> ToView() const { return std::basic_string_view<CharType>(Data(), Size()); } + std::filesystem::path ToPath() const { return std::filesystem::path(ToView()); } + + std::string ToUtf8() const + { +#if ZEN_PLATFORM_WINDOWS + return WideToUtf8(ToView()); +#else + return std::string(ToView()); +#endif + } + + void AppendSeparator() + { + if (ToView().ends_with(std::filesystem::path::preferred_separator) +#if ZEN_PLATFORM_WINDOWS + || ToView().ends_with('/') +#endif + ) + return; + + Super::Append(std::filesystem::path::preferred_separator); + } +}; + +template<size_t N> +class PathBuilder : public PathBuilderBase +{ +public: + PathBuilder() { Init(m_Buffer, N); } + +private: + PathBuilderBase::CharType m_Buffer[N]; +}; + +template<size_t N> +class ExtendablePathBuilder : public PathBuilder<N> +{ +public: + ExtendablePathBuilder() { this->m_IsExtendable = true; } +}; + +struct DiskSpace +{ + uint64_t Free{}; + uint64_t Total{}; +}; + +ZENCORE_API DiskSpace DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error); + +inline bool +DiskSpaceInfo(std::filesystem::path Directory, DiskSpace& Space) +{ + std::error_code Err; + Space = DiskSpaceInfo(Directory, Err); + return !Err; +} + +/** + * Efficient file system traversal + * + * Uses the best available mechanism for the platform in question and could take + * advantage of any file system tracking mechanisms in the future + * + */ +class FileSystemTraversal +{ +public: + struct TreeVisitor + { + using path_view = std::basic_string_view<std::filesystem::path::value_type>; + using path_string = std::filesystem::path::string_type; + + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0; + + // This should return true if we should recurse into the directory + virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0; + }; + + void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor); +}; + +struct DirectoryContent +{ + static const uint8_t IncludeDirsFlag = 1u << 0; + static const uint8_t IncludeFilesFlag = 1u << 1; + static const uint8_t RecursiveFlag = 1u << 2; + std::vector<std::filesystem::path> Files; + std::vector<std::filesystem::path> Directories; +}; + +void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent); + +std::string GetEnvVariable(std::string_view VariableName); + +////////////////////////////////////////////////////////////////////////// + +void filesystem_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/fmtutils.h b/src/zencore/include/zencore/fmtutils.h new file mode 100644 index 000000000..70867fe72 --- /dev/null +++ b/src/zencore/include/zencore/fmtutils.h @@ -0,0 +1,52 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/iohash.h> +#include <zencore/string.h> +#include <zencore/uid.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <fmt/format.h> +ZEN_THIRD_PARTY_INCLUDES_END + +#include <filesystem> +#include <string_view> + +// Custom formatting for some zencore types + +template<> +struct fmt::formatter<zen::IoHash> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const zen::IoHash& Hash, FormatContext& ctx) + { + zen::IoHash::String_t String; + Hash.ToHexString(String); + return formatter<string_view>::format({String, zen::IoHash::StringLength}, ctx); + } +}; + +template<> +struct fmt::formatter<zen::Oid> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const zen::Oid& Id, FormatContext& ctx) + { + zen::StringBuilder<32> String; + Id.ToString(String); + return formatter<string_view>::format({String.c_str(), zen::Oid::StringLength}, ctx); + } +}; + +template<> +struct fmt::formatter<std::filesystem::path> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const std::filesystem::path& Path, FormatContext& ctx) + { + zen::ExtendableStringBuilder<128> String; + String << Path.u8string(); + return formatter<string_view>::format(String.ToView(), ctx); + } +}; diff --git a/src/zencore/include/zencore/intmath.h b/src/zencore/include/zencore/intmath.h new file mode 100644 index 000000000..f24caed6e --- /dev/null +++ b/src/zencore/include/zencore/intmath.h @@ -0,0 +1,183 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <stdint.h> + +////////////////////////////////////////////////////////////////////////// + +#if ZEN_COMPILER_MSC || ZEN_PLATFORM_WINDOWS +# pragma intrinsic(_BitScanReverse) +# pragma intrinsic(_BitScanReverse64) +#else +inline uint8_t +_BitScanReverse(unsigned long* Index, uint32_t Mask) +{ + if (Mask == 0) + { + return 0; + } + + *Index = 31 - __builtin_clz(Mask); + return 1; +} + +inline uint8_t +_BitScanReverse64(unsigned long* Index, uint64_t Mask) +{ + if (Mask == 0) + { + return 0; + } + + *Index = 63 - __builtin_clzll(Mask); + return 1; +} + +inline uint8_t +_BitScanForward64(unsigned long* Index, uint64_t Mask) +{ + if (Mask == 0) + { + return 0; + } + + *Index = __builtin_ctzll(Mask); + return 1; +} +#endif + +namespace zen { + +inline constexpr bool +IsPow2(uint64_t n) +{ + return 0 == (n & (n - 1)); +} + +/// Round an integer up to the closest integer multiplier of 'base' ('base' must be a power of two) +template<Integral T> +T +RoundUp(T Value, auto Base) +{ + ZEN_ASSERT_SLOW(IsPow2(Base)); + return ((Value + T(Base - 1)) & (~T(Base - 1))); +} + +bool +IsMultipleOf(Integral auto Value, auto MultiplierPow2) +{ + ZEN_ASSERT_SLOW(IsPow2(MultiplierPow2)); + return (Value & (MultiplierPow2 - 1)) == 0; +} + +inline uint64_t +NextPow2(uint64_t n) +{ + // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + + --n; + + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n |= n >> 32; + + return n + 1; +} + +static inline uint32_t +FloorLog2(uint32_t Value) +{ + // Use BSR to return the log2 of the integer + unsigned long Log2; + if (_BitScanReverse(&Log2, Value) != 0) + { + return Log2; + } + + return 0; +} + +static inline uint32_t +CountLeadingZeros(uint32_t Value) +{ + unsigned long Log2 = 0; + _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1); + return 32 - Log2; +} + +static inline uint64_t +FloorLog2_64(uint64_t Value) +{ + unsigned long Log2 = 0; + long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); + return Log2 & Mask; +} + +static inline uint64_t +CountLeadingZeros64(uint64_t Value) +{ + unsigned long Log2 = 0; + long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); + return ((63 - Log2) & Mask) | (64 & ~Mask); +} + +static inline uint64_t +CeilLogTwo64(uint64_t Arg) +{ + int64_t Bitmask = ((int64_t)(CountLeadingZeros64(Arg) << 57)) >> 63; + return (64 - CountLeadingZeros64(Arg - 1)) & (~Bitmask); +} + +static inline uint64_t +CountTrailingZeros64(uint64_t Value) +{ + if (Value == 0) + { + return 64; + } + unsigned long BitIndex; // 0-based, where the LSB is 0 and MSB is 31 + _BitScanForward64(&BitIndex, Value); // Scans from LSB to MSB + return BitIndex; +} + +////////////////////////////////////////////////////////////////////////// + +static inline bool +IsPointerAligned(const void* Ptr, uint64_t Alignment) +{ + ZEN_ASSERT_SLOW(IsPow2(Alignment)); + + return 0 == (reinterpret_cast<uintptr_t>(Ptr) & (Alignment - 1)); +} + +////////////////////////////////////////////////////////////////////////// + +#if ZEN_PLATFORM_WINDOWS +# ifdef min +# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead" +# endif +#endif + +constexpr auto +Min(auto x, auto y) +{ + return x < y ? x : y; +} + +constexpr auto +Max(auto x, auto y) +{ + return x > y ? x : y; +} + +////////////////////////////////////////////////////////////////////////// + +void intmath_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h new file mode 100644 index 000000000..a39dbf6d6 --- /dev/null +++ b/src/zencore/include/zencore/iobuffer.h @@ -0,0 +1,423 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <memory.h> +#include <zencore/memory.h> +#include <atomic> +#include "refcount.h" +#include "zencore.h" + +#include <filesystem> + +namespace zen { + +struct IoHash; +struct IoBufferExtendedCore; + +enum class ZenContentType : uint8_t +{ + kBinary = 0, // Note that since this is zero, this will be the default value in IoBuffer + kText = 1, + kJSON = 2, + kCbObject = 3, + kCbPackage = 4, + kYAML = 5, + kCbPackageOffer = 6, + kCompressedBinary = 7, + kUnknownContentType = 8, + kHTML = 9, + kJavaScript = 10, + kCSS = 11, + kPNG = 12, + kIcon = 13, + kCOUNT +}; + +inline std::string_view +ToString(ZenContentType ContentType) +{ + using namespace std::literals; + + switch (ContentType) + { + default: + case ZenContentType::kUnknownContentType: + return "unknown"sv; + case ZenContentType::kBinary: + return "binary"sv; + case ZenContentType::kText: + return "text"sv; + case ZenContentType::kJSON: + return "json"sv; + case ZenContentType::kCbObject: + return "cb-object"sv; + case ZenContentType::kCbPackage: + return "cb-package"sv; + case ZenContentType::kCbPackageOffer: + return "cb-package-offer"sv; + case ZenContentType::kCompressedBinary: + return "compressed-binary"sv; + case ZenContentType::kYAML: + return "yaml"sv; + case ZenContentType::kHTML: + return "html"sv; + case ZenContentType::kJavaScript: + return "javascript"sv; + case ZenContentType::kCSS: + return "css"sv; + case ZenContentType::kPNG: + return "png"sv; + case ZenContentType::kIcon: + return "icon"sv; + } +} + +struct IoBufferFileReference +{ + void* FileHandle; + uint64_t FileChunkOffset; + uint64_t FileChunkSize; +}; + +struct IoBufferCore +{ +public: + inline IoBufferCore() : m_Flags(kIsNull) {} + inline IoBufferCore(const void* DataPtr, size_t SizeBytes) : m_DataPtr(DataPtr), m_DataBytes(SizeBytes) {} + inline IoBufferCore(const IoBufferCore* Outer, const void* DataPtr, size_t SizeBytes) + : m_DataPtr(DataPtr) + , m_DataBytes(SizeBytes) + , m_OuterCore(Outer) + { + } + + ZENCORE_API explicit IoBufferCore(size_t SizeBytes); + ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment); + ZENCORE_API ~IoBufferCore(); + + // Reference counting + + inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); } + inline uint32_t Release() const + { + const uint32_t NewRefCount = AtomicDecrement(const_cast<IoBufferCore*>(this)->m_RefCount); + if (NewRefCount == 0) + { + DeleteThis(); + } + return NewRefCount; + } + + // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it + + IoBufferCore(const IoBufferCore&) = delete; + IoBufferCore(IoBufferCore&&) = delete; + IoBufferCore& operator=(const IoBufferCore&) = delete; + IoBufferCore& operator=(IoBufferCore&&) = delete; + + // + + ZENCORE_API void Materialize() const; + ZENCORE_API void DeleteThis() const; + ZENCORE_API void MakeOwned(bool Immutable = true); + + inline void EnsureDataValid() const + { + const uint32_t LocalFlags = m_Flags.load(std::memory_order_acquire); + if ((LocalFlags & kIsExtended) && !(LocalFlags & kIsMaterialized)) + { + Materialize(); + } + } + + inline bool IsOwnedByThis() const { return !!(m_Flags.load(std::memory_order_relaxed) & kIsOwnedByThis); } + + inline void SetIsOwnedByThis(bool NewState) + { + if (NewState) + { + m_Flags.fetch_or(kIsOwnedByThis, std::memory_order_relaxed); + } + else + { + m_Flags.fetch_and(~kIsOwnedByThis, std::memory_order_relaxed); + } + } + + inline bool IsOwned() const + { + if (IsOwnedByThis()) + { + return true; + } + return m_OuterCore && m_OuterCore->IsOwned(); + } + + inline bool IsImmutable() const { return (m_Flags.load(std::memory_order_relaxed) & kIsMutable) == 0; } + inline bool IsWholeFile() const { return (m_Flags.load(std::memory_order_relaxed) & kIsWholeFile) != 0; } + inline bool IsNull() const { return (m_Flags.load(std::memory_order_relaxed) & kIsNull) != 0; } + + inline IoBufferExtendedCore* ExtendedCore(); + inline const IoBufferExtendedCore* ExtendedCore() const; + + ZENCORE_API void* MutableDataPointer() const; + + inline const void* DataPointer() const + { + EnsureDataValid(); + return m_DataPtr; + } + + inline size_t DataBytes() const { return m_DataBytes; } + + inline void Set(const void* Ptr, size_t Sz) + { + m_DataPtr = Ptr; + m_DataBytes = Sz; + } + + inline void SetIsImmutable(bool NewState) + { + if (!NewState) + { + m_Flags.fetch_or(kIsMutable, std::memory_order_relaxed); + } + else + { + m_Flags.fetch_and(~kIsMutable, std::memory_order_relaxed); + } + } + + inline void SetIsWholeFile(bool NewState) + { + if (NewState) + { + m_Flags.fetch_or(kIsWholeFile, std::memory_order_relaxed); + } + else + { + m_Flags.fetch_and(~kIsWholeFile, std::memory_order_relaxed); + } + } + + inline void SetContentType(ZenContentType ContentType) + { + ZEN_ASSERT_SLOW((uint32_t(ContentType) & kContentTypeMask) == uint32_t(ContentType)); + uint32_t OldValue = m_Flags.load(std::memory_order_relaxed); + uint32_t NewValue; + do + { + NewValue = (OldValue & ~(kContentTypeMask << kContentTypeShift)) | (uint32_t(ContentType) << kContentTypeShift); + } while (!m_Flags.compare_exchange_weak(OldValue, NewValue, std::memory_order_relaxed, std::memory_order_relaxed)); + } + + inline ZenContentType GetContentType() const + { + return ZenContentType((m_Flags.load(std::memory_order_relaxed) >> kContentTypeShift) & kContentTypeMask); + } + + inline uint32_t GetRefCount() const { return m_RefCount; } + +protected: + uint32_t m_RefCount = 0; + mutable std::atomic<uint32_t> m_Flags{0}; + mutable const void* m_DataPtr = nullptr; + size_t m_DataBytes = 0; + RefPtr<const IoBufferCore> m_OuterCore; + + enum + { + kContentTypeShift = 24, + kContentTypeMask = 0xf + }; + + static_assert((uint32_t(ZenContentType::kUnknownContentType) & ~kContentTypeMask) == 0); + + enum Flags : uint32_t + { + kIsNull = 1 << 0, // This is a null IoBuffer + kIsMutable = 1 << 1, + kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore + kIsMaterialized = 1 << 3, // Data pointers are valid + kLowLevelAlloc = 1 << 4, // Using direct memory allocation + kIsWholeFile = 1 << 5, // References an entire file + kIoBufferAlloc = 1 << 6, // Using IoBuffer allocator + kIsOwnedByThis = 1 << 7, + + // Note that we have some extended flags defined below + // so not all bits are available to use here + + kContentTypeBit0 = 1 << (24 + 0), // These constants + kContentTypeBit1 = 1 << (24 + 1), // are here mostly to + kContentTypeBit2 = 1 << (24 + 2), // indicate that these + kContentTypeBit3 = 1 << (24 + 3), // bits are reserved + }; + + void AllocateBuffer(size_t InSize, size_t Alignment) const; + void FreeBuffer(); +}; + +/** + * An "Extended" core references a segment of a file + */ + +struct IoBufferExtendedCore : public IoBufferCore +{ + IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership); + IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size); + ~IoBufferExtendedCore(); + + enum ExtendedFlags + { + kOwnsFile = 1 << 16, + kOwnsMmap = 1 << 17 + }; + + void Materialize() const; + bool GetFileReference(IoBufferFileReference& OutRef) const; + void MarkAsDeleteOnClose(); + +private: + void* m_FileHandle = nullptr; + uint64_t m_FileOffset = 0; + mutable void* m_MmapHandle = nullptr; + mutable void* m_MappedPointer = nullptr; + bool m_DeleteOnClose = false; +}; + +inline IoBufferExtendedCore* +IoBufferCore::ExtendedCore() +{ + if (m_Flags.load(std::memory_order_relaxed) & kIsExtended) + { + return static_cast<IoBufferExtendedCore*>(this); + } + + return nullptr; +} + +inline const IoBufferExtendedCore* +IoBufferCore::ExtendedCore() const +{ + if (m_Flags.load(std::memory_order_relaxed) & kIsExtended) + { + return static_cast<const IoBufferExtendedCore*>(this); + } + + return nullptr; +} + +/** + * I/O buffer + * + * This represents a reference to a payload in memory or on disk + * + */ +class IoBuffer +{ +public: + enum ECloneTag + { + Clone + }; + enum EWrapTag + { + Wrap + }; + enum EFileTag + { + File + }; + enum EBorrowedFileTag + { + BorrowedFile + }; + + inline IoBuffer() = default; + inline IoBuffer(IoBuffer&& Rhs) noexcept = default; + inline IoBuffer(const IoBuffer& Rhs) = default; + inline IoBuffer& operator=(const IoBuffer& Rhs) = default; + inline IoBuffer& operator=(IoBuffer&& Rhs) noexcept = default; + + /** Create an uninitialized buffer of the given size + */ + ZENCORE_API explicit IoBuffer(size_t InSize); + + /** Create an uninitialized buffer of the given size with the specified alignment + */ + ZENCORE_API explicit IoBuffer(size_t InSize, uint64_t InAlignment); + + /** Create a buffer which references a sequence of bytes inside another buffer + */ + ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes = ~0ull); + + /** Create a buffer which references a range of bytes which we assume will live + * for the entire life time. + */ + inline IoBuffer(EWrapTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(DataPtr, SizeBytes)) {} + + inline IoBuffer(ECloneTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(SizeBytes)) + { + memcpy(const_cast<void*>(m_Core->DataPointer()), DataPtr, SizeBytes); + } + + ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); + ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); + + inline explicit operator bool() const { return !m_Core->IsNull(); } + inline operator MemoryView() const& { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); } + inline void MakeOwned() { return m_Core->MakeOwned(); } + [[nodiscard]] inline bool IsOwned() const { return m_Core->IsOwned(); } + [[nodiscard]] inline bool IsWholeFile() const { return m_Core->IsWholeFile(); } + [[nodiscard]] void* MutableData() const { return m_Core->MutableDataPointer(); } + void MakeImmutable() { m_Core->SetIsImmutable(true); } + [[nodiscard]] const void* Data() const { return m_Core->DataPointer(); } + [[nodiscard]] const void* GetData() const { return m_Core->DataPointer(); } + [[nodiscard]] size_t Size() const { return m_Core->DataBytes(); } + [[nodiscard]] size_t GetSize() const { return m_Core->DataBytes(); } + inline void SetContentType(ZenContentType ContentType) { m_Core->SetContentType(ContentType); } + [[nodiscard]] inline ZenContentType GetContentType() const { return m_Core->GetContentType(); } + [[nodiscard]] ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const; + void MarkAsDeleteOnClose(); + + inline MemoryView GetView() const { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); } + inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Core->MutableDataPointer(), m_Core->DataBytes()); } + + template<typename T> + [[nodiscard]] const T* Data() const + { + return reinterpret_cast<const T*>(m_Core->DataPointer()); + } + + template<typename T> + [[nodiscard]] T* MutableData() const + { + return reinterpret_cast<T*>(m_Core->MutableDataPointer()); + } + +private: + RefPtr<IoBufferCore> m_Core = new IoBufferCore; + + IoBuffer(IoBufferCore* Core) : m_Core(Core) {} + + friend class SharedBuffer; + friend class IoBufferBuilder; +}; + +class IoBufferBuilder +{ +public: + ZENCORE_API static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); + ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer ReadFromFileMaybe(IoBuffer& InBuffer); + inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); } + inline static IoBuffer MakeCloneFromMemory(MemoryView Memory) { return IoBuffer(IoBuffer::Clone, Memory.GetData(), Memory.GetSize()); } +}; + +IoHash HashBuffer(IoBuffer& Buffer); + +void iobuffer_forcelink(); + +} // namespace zen diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h new file mode 100644 index 000000000..fd0f4b2a7 --- /dev/null +++ b/src/zencore/include/zencore/iohash.h @@ -0,0 +1,115 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/blake3.h> +#include <zencore/memory.h> + +#include <compare> +#include <string_view> + +namespace zen { + +class StringBuilderBase; +class CompositeBuffer; + +/** + * Hash used for content addressable storage + * + * This is basically a BLAKE3-160 hash (note: this is probably not an officially + * recognized identifier). It is generated by computing a 32-byte BLAKE3 hash and + * picking the first 20 bytes of the resulting hash. + * + */ +struct IoHash +{ + alignas(uint32_t) uint8_t Hash[20] = {}; + + static IoHash MakeFrom(const void* data /* 20 bytes */) + { + IoHash Io; + memcpy(Io.Hash, data, sizeof Io); + return Io; + } + + static IoHash FromBLAKE3(const BLAKE3& Blake3) + { + IoHash Io; + memcpy(Io.Hash, Blake3.Hash, sizeof Io.Hash); + return Io; + } + + static IoHash HashBuffer(const void* data, size_t byteCount); + static IoHash HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); } + static IoHash HashBuffer(const CompositeBuffer& Buffer); + static IoHash FromHexString(const char* string); + static IoHash FromHexString(const std::string_view string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + std::string ToHexString() const; + + static const int StringLength = 40; + typedef char String_t[StringLength + 1]; + + static const IoHash Zero; // Initialized to all zeros + + inline auto operator<=>(const IoHash& rhs) const = default; + + struct Hasher + { + size_t operator()(const IoHash& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct IoHashStream +{ + /// Begin streaming hash compute (not needed on freshly constructed instance) + void Reset() { m_Blake3Stream.Reset(); } + + /// Append another chunk + IoHashStream& Append(const void* data, size_t byteCount) + { + m_Blake3Stream.Append(data, byteCount); + return *this; + } + + /// Append another chunk + IoHashStream& Append(MemoryView Data) + { + m_Blake3Stream.Append(Data.GetData(), Data.GetSize()); + return *this; + } + + /// Obtain final hash. If you wish to reuse the instance call reset() + IoHash GetHash() + { + BLAKE3 b3 = m_Blake3Stream.GetHash(); + + IoHash Io; + memcpy(Io.Hash, b3.Hash, sizeof Io.Hash); + + return Io; + } + +private: + BLAKE3Stream m_Blake3Stream; +}; + +} // namespace zen + +namespace std { + +template<> +struct hash<zen::IoHash> : public zen::IoHash::Hasher +{ +}; + +} // namespace std diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h new file mode 100644 index 000000000..5cbe034cf --- /dev/null +++ b/src/zencore/include/zencore/logging.h @@ -0,0 +1,136 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <spdlog/spdlog.h> +#undef GetObject +ZEN_THIRD_PARTY_INCLUDES_END + +#include <string_view> + +namespace zen::logging { + +spdlog::logger& Default(); +void SetDefault(std::shared_ptr<spdlog::logger> NewDefaultLogger); +spdlog::logger& ConsoleLog(); +spdlog::logger& Get(std::string_view Name); +spdlog::logger* ErrorLog(); +void SetErrorLog(std::shared_ptr<spdlog::logger>&& NewErrorLogger); + +void InitializeLogging(); +void ShutdownLogging(); + +} // namespace zen::logging + +namespace zen { +extern spdlog::logger* TheDefaultLogger; + +inline spdlog::logger& +Log() +{ + return *TheDefaultLogger; +} + +using logging::ConsoleLog; +using logging::ErrorLog; +} // namespace zen + +using zen::ConsoleLog; +using zen::ErrorLog; +using zen::Log; + +struct LogCategory +{ + LogCategory(std::string_view InCategory) : Category(InCategory) {} + + spdlog::logger& Logger() + { + static spdlog::logger& Inst = zen::logging::Get(Category); + return Inst; + } + + std::string Category; +}; + +inline consteval bool +LogIsErrorLevel(int level) +{ + return (level == spdlog::level::err || level == spdlog::level::critical); +}; + +#define ZEN_LOG_WITH_LOCATION(logger, loc, level, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + if (logger.should_log(level)) \ + { \ + logger.log(loc, level, fmtstr, ##__VA_ARGS__); \ + if (LogIsErrorLevel(level)) \ + { \ + if (auto ErrLogger = zen::logging::ErrorLog(); ErrLogger != nullptr) \ + { \ + ErrLogger->log(loc, level, fmtstr, ##__VA_ARGS__); \ + } \ + } \ + } \ + } while (false); + +#define ZEN_LOG(logger, level, fmtstr, ...) ZEN_LOG_WITH_LOCATION(logger, spdlog::source_loc{}, level, fmtstr, ##__VA_ARGS__) + +#define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \ + static struct LogCategory##Category : public LogCategory \ + { \ + LogCategory##Category() : LogCategory(Name) {} \ + } Category; + +#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::trace, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::debug, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::info, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::warn, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_LOG_ERROR(Category, fmtstr, ...) \ + ZEN_LOG_WITH_LOCATION(Category.Logger(), \ + spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \ + spdlog::level::err, \ + fmtstr##sv, \ + ##__VA_ARGS__) + +#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) \ + ZEN_LOG_WITH_LOCATION(Category.Logger(), \ + spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \ + spdlog::level::critical, \ + fmtstr##sv, \ + ##__VA_ARGS__) + + // Helper macros for logging + +#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::trace, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::debug, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::info, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::warn, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_ERROR(fmtstr, ...) \ + ZEN_LOG_WITH_LOCATION(Log(), spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), spdlog::level::err, fmtstr##sv, ##__VA_ARGS__) + +#define ZEN_CRITICAL(fmtstr, ...) \ + ZEN_LOG_WITH_LOCATION(Log(), \ + spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \ + spdlog::level::critical, \ + fmtstr##sv, \ + ##__VA_ARGS__) + +#define ZEN_CONSOLE(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + ConsoleLog().info(fmtstr##sv, ##__VA_ARGS__); \ + } while (false) diff --git a/src/zencore/include/zencore/md5.h b/src/zencore/include/zencore/md5.h new file mode 100644 index 000000000..d934dd86b --- /dev/null +++ b/src/zencore/include/zencore/md5.h @@ -0,0 +1,50 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <stdint.h> +#include <compare> +#include "zencore.h" + +namespace zen { + +class StringBuilderBase; + +struct MD5 +{ + uint8_t Hash[16]; + + inline auto operator<=>(const MD5& rhs) const = default; + + static const int StringLength = 32; + typedef char String_t[StringLength + 1]; + + static MD5 HashMemory(const void* data, size_t byteCount); + static MD5 FromHexString(const char* string); + const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static MD5 Zero; // Initialized to all zeroes +}; + +/** + * Utility class for computing MD5 hashes + */ +class MD5Stream +{ +public: + MD5Stream(); + + /// Begin streaming MD5 compute (not needed on freshly constructed MD5Stream instance) + void Reset(); + /// Append another chunk + MD5Stream& Append(const void* data, size_t byteCount); + /// Obtain final MD5 hash. If you wish to reuse the MD5Stream instance call reset() + MD5 GetHash(); + +private: +}; + +void md5_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/memory.h b/src/zencore/include/zencore/memory.h new file mode 100644 index 000000000..560fa9ffc --- /dev/null +++ b/src/zencore/include/zencore/memory.h @@ -0,0 +1,401 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/intmath.h> +#include <zencore/thread.h> + +#include <cstddef> +#include <cstring> +#include <span> +#include <vector> + +namespace zen { + +#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 201911L +template<typename T> +concept ContiguousRange = std::ranges::contiguous_range<T>; +#else +template<typename T> +concept ContiguousRange = true; +#endif + +struct MemoryView; + +class MemoryArena +{ +public: + ZENCORE_API MemoryArena(); + ZENCORE_API ~MemoryArena(); + + ZENCORE_API void* Alloc(size_t Size, size_t Alignment); + ZENCORE_API void Free(void* Ptr); + +private: +}; + +class Memory +{ +public: + ZENCORE_API static void* Alloc(size_t Size, size_t Alignment = sizeof(void*)); + ZENCORE_API static void Free(void* Ptr); +}; + +/** Allocator which claims fixed-size blocks from the underlying allocator. + + There is no way to free individual memory blocks. + + \note This is not thread-safe, you will need to provide synchronization yourself +*/ + +class ChunkingLinearAllocator +{ +public: + ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment = sizeof(std::max_align_t)); + ~ChunkingLinearAllocator(); + + ZENCORE_API void Reset(); + + ZENCORE_API void* Alloc(size_t Size, size_t Alignment = sizeof(void*)); + inline void Free(void* Ptr) { ZEN_UNUSED(Ptr); /* no-op */ } + + ChunkingLinearAllocator(const ChunkingLinearAllocator&) = delete; + ChunkingLinearAllocator& operator=(const ChunkingLinearAllocator&) = delete; + +private: + uint8_t* m_ChunkCursor = nullptr; + uint64_t m_ChunkBytesRemain = 0; + const uint64_t m_ChunkSize = 0; + const uint64_t m_ChunkAlignment = 0; + std::vector<void*> m_ChunkList; +}; + +////////////////////////////////////////////////////////////////////////// + +struct MutableMemoryView +{ + MutableMemoryView() = default; + + MutableMemoryView(void* DataPtr, size_t DataSize) + : m_Data(reinterpret_cast<uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<uint8_t*>(DataPtr) + DataSize) + { + } + + MutableMemoryView(void* DataPtr, void* DataEndPtr) + : m_Data(reinterpret_cast<uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<uint8_t*>(DataEndPtr)) + { + } + + inline bool IsEmpty() const { return m_Data == m_DataEnd; } + void* GetData() const { return m_Data; } + void* GetDataEnd() const { return m_DataEnd; } + size_t GetSize() const { return reinterpret_cast<uint8_t*>(m_DataEnd) - reinterpret_cast<uint8_t*>(m_Data); } + + inline bool EqualBytes(const MutableMemoryView& InView) const + { + const size_t Size = GetSize(); + + return Size == InView.GetSize() && (memcmp(m_Data, InView.m_Data, Size) == 0); + } + + /** Modifies the view to be the given number of bytes from the right. */ + inline void RightInline(uint64_t InSize) + { + const uint64_t OldSize = GetSize(); + const uint64_t NewSize = zen::Min(OldSize, InSize); + m_Data = GetDataAtOffsetNoCheck(OldSize - NewSize); + m_DataEnd = m_Data + NewSize; + } + + /** Returns the right-most part of the view by taking the given number of bytes from the right. */ + [[nodiscard]] inline MutableMemoryView Right(uint64_t InSize) const + { + MutableMemoryView View(*this); + View.RightChopInline(InSize); + return View; + } + + /** Modifies the view by chopping the given number of bytes from the left. */ + inline void RightChopInline(uint64_t InSize) + { + const uint64_t Offset = zen::Min(GetSize(), InSize); + m_Data = GetDataAtOffsetNoCheck(Offset); + } + + /** Returns the left-most part of the view by taking the given number of bytes from the left. */ + constexpr inline MutableMemoryView Left(uint64_t InSize) const + { + MutableMemoryView View(*this); + View.LeftInline(InSize); + return View; + } + + /** Modifies the view to be the given number of bytes from the left. */ + constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = zen::Min(m_DataEnd, m_Data + InSize); } + + /** Modifies the view to be the middle part by taking up to the given number of bytes from the given offset. */ + inline void MidInline(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) + { + RightChopInline(InOffset); + LeftInline(InSize); + } + + /** Returns the middle part of the view by taking up to the given number of bytes from the given position. */ + [[nodiscard]] inline MutableMemoryView Mid(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) const + { + MutableMemoryView View(*this); + View.MidInline(InOffset, InSize); + return View; + } + + /** Returns the right-most part of the view by chopping the given number of bytes from the left. */ + [[nodiscard]] inline MutableMemoryView RightChop(uint64_t InSize) const + { + MutableMemoryView View(*this); + View.RightChopInline(InSize); + return View; + } + + inline MutableMemoryView& operator+=(size_t InSize) + { + RightChopInline(InSize); + return *this; + } + + /** Copies bytes from the input view into this view, and returns the remainder of this view. */ + inline MutableMemoryView CopyFrom(MemoryView InView) const; + +private: + uint8_t* m_Data = nullptr; + uint8_t* m_DataEnd = nullptr; + + /** Returns the data pointer advanced by an offset in bytes. */ + inline constexpr uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; } +}; + +////////////////////////////////////////////////////////////////////////// + +struct MemoryView +{ + MemoryView() = default; + + MemoryView(const MutableMemoryView& MutableView) + : m_Data(reinterpret_cast<const uint8_t*>(MutableView.GetData())) + , m_DataEnd(m_Data + MutableView.GetSize()) + { + } + + MemoryView(const void* DataPtr, size_t DataSize) + : m_Data(reinterpret_cast<const uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<const uint8_t*>(DataPtr) + DataSize) + { + } + + MemoryView(const void* DataPtr, const void* DataEndPtr) + : m_Data(reinterpret_cast<const uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<const uint8_t*>(DataEndPtr)) + { + } + + inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); } + inline bool IsEmpty() const { return m_Data == m_DataEnd; } + const void* GetData() const { return m_Data; } + const void* GetDataEnd() const { return m_DataEnd; } + size_t GetSize() const { return reinterpret_cast<const uint8_t*>(m_DataEnd) - reinterpret_cast<const uint8_t*>(m_Data); } + inline bool operator==(const MemoryView& Rhs) const { return m_Data == Rhs.m_Data && m_DataEnd == Rhs.m_DataEnd; } + + inline bool EqualBytes(const MemoryView& InView) const + { + const size_t Size = GetSize(); + + return Size == InView.GetSize() && (memcmp(m_Data, InView.GetData(), Size) == 0); + } + + inline MemoryView& operator+=(size_t InSize) + { + RightChopInline(InSize); + return *this; + } + + /** Modifies the view by chopping the given number of bytes from the left. */ + inline void RightChopInline(uint64_t InSize) + { + const uint64_t Offset = zen::Min(GetSize(), InSize); + m_Data = GetDataAtOffsetNoCheck(Offset); + } + + inline MemoryView RightChop(uint64_t InSize) + { + MemoryView View(*this); + View.RightChopInline(InSize); + return View; + } + + /** Returns the right-most part of the view by taking the given number of bytes from the right. */ + [[nodiscard]] inline MemoryView Right(uint64_t InSize) const + { + MemoryView View(*this); + View.RightInline(InSize); + return View; + } + + /** Modifies the view to be the given number of bytes from the right. */ + inline void RightInline(uint64_t InSize) + { + const uint64_t OldSize = GetSize(); + const uint64_t NewSize = zen::Min(OldSize, InSize); + m_Data = GetDataAtOffsetNoCheck(OldSize - NewSize); + m_DataEnd = m_Data + NewSize; + } + + /** Returns the left-most part of the view by taking the given number of bytes from the left. */ + inline MemoryView Left(uint64_t InSize) const + { + MemoryView View(*this); + View.LeftInline(InSize); + return View; + } + + /** Modifies the view to be the given number of bytes from the left. */ + inline void LeftInline(uint64_t InSize) + { + InSize = zen::Min(GetSize(), InSize); + m_DataEnd = zen::Min(m_DataEnd, m_Data + InSize); + } + + /** Modifies the view to be the middle part by taking up to the given number of bytes from the given offset. */ + inline void MidInline(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) + { + RightChopInline(InOffset); + LeftInline(InSize); + } + + /** Returns the middle part of the view by taking up to the given number of bytes from the given position. */ + [[nodiscard]] inline MemoryView Mid(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) const + { + MemoryView View(*this); + View.MidInline(InOffset, InSize); + return View; + } + + constexpr void Reset() + { + m_Data = nullptr; + m_DataEnd = nullptr; + } + +private: + const uint8_t* m_Data = nullptr; + const uint8_t* m_DataEnd = nullptr; + + /** Returns the data pointer advanced by an offset in bytes. */ + inline constexpr const uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; } +}; + +inline MutableMemoryView +MutableMemoryView::CopyFrom(MemoryView InView) const +{ + ZEN_ASSERT(InView.GetSize() <= GetSize()); + memcpy(m_Data, InView.GetData(), InView.GetSize()); + return RightChop(InView.GetSize()); +} + +/** Advances the start of the view by an offset, which is clamped to stay within the view. */ +inline MemoryView +operator+(const MemoryView& View, uint64_t Offset) +{ + return MemoryView(View) += Offset; +} + +/** Advances the start of the view by an offset, which is clamped to stay within the view. */ +inline MemoryView +operator+(uint64_t Offset, const MemoryView& View) +{ + return MemoryView(View) += Offset; +} + +/** Advances the start of the view by an offset, which is clamped to stay within the view. */ +inline MutableMemoryView +operator+(const MutableMemoryView& View, uint64_t Offset) +{ + return MutableMemoryView(View) += Offset; +} + +/** Advances the start of the view by an offset, which is clamped to stay within the view. */ +inline MutableMemoryView +operator+(uint64_t Offset, const MutableMemoryView& View) +{ + return MutableMemoryView(View) += Offset; +} + +/** + * Make a non-owning view of the memory of the initializer list. + * + * This overload is only available when the element type does not need to be deduced. + */ +template<typename T> +[[nodiscard]] inline MemoryView +MakeMemoryView(std::initializer_list<typename std::type_identity<T>::type> List) +{ + return MemoryView(List.begin(), List.size() * sizeof(T)); +} + +/** Make a non-owning view of the memory of the contiguous container. */ +template<ContiguousRange R> +[[nodiscard]] constexpr inline MemoryView +MakeMemoryView(const R& Container) +{ + std::span Span = Container; + return MemoryView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type)); +} + +/** Make a non-owning const view starting at Data and ending at DataEnd. */ + +[[nodiscard]] inline MemoryView +MakeMemoryView(const void* Data, const void* DataEnd) +{ + return MemoryView(Data, DataEnd); +} + +[[nodiscard]] inline MemoryView +MakeMemoryView(const void* Data, uint64_t Size) +{ + return MemoryView(Data, reinterpret_cast<const uint8_t*>(Data) + Size); +} + +/** + * Make a non-owning mutable view of the memory of the initializer list. + * + * This overload is only available when the element type does not need to be deduced. + */ +template<typename T> +[[nodiscard]] inline MutableMemoryView +MakeMutableMemoryView(std::initializer_list<typename std::type_identity<T>::type> List) +{ + return MutableMemoryView(List.begin(), List.size() * sizeof(T)); +} + +/** Make a non-owning mutable view of the memory of the contiguous container. */ +template<ContiguousRange R> +[[nodiscard]] constexpr inline MutableMemoryView +MakeMutableMemoryView(R& Container) +{ + std::span Span = Container; + return MutableMemoryView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type)); +} + +/** Make a non-owning mutable view starting at Data and ending at DataEnd. */ + +[[nodiscard]] inline MutableMemoryView +MakeMutableMemoryView(void* Data, void* DataEnd) +{ + return MutableMemoryView(Data, DataEnd); +} + +void memory_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/meta.h b/src/zencore/include/zencore/meta.h new file mode 100644 index 000000000..82eb5cc30 --- /dev/null +++ b/src/zencore/include/zencore/meta.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +/* This file contains utility functions for meta programming + * + * Since you're in here you're probably quite observant, and you'll + * note that it's quite barren here. This is because template + * metaprogramming is awful and I try not to engage in it. However, + * sometimes these things are forced upon us. + * + */ + +namespace zen { + +/** + * Uses implicit conversion to create an instance of a specific type. + * Useful to make things clearer or circumvent unintended type deduction in templates. + * Safer than C casts and static_casts, e.g. does not allow down-casts + * + * @param Obj The object (usually pointer or reference) to convert. + * + * @return The object converted to the specified type. + */ +template<typename T> +inline T +ImplicitConv(typename std::type_identity<T>::type Obj) +{ + return Obj; +} + +} // namespace zen diff --git a/src/zencore/include/zencore/mpscqueue.h b/src/zencore/include/zencore/mpscqueue.h new file mode 100644 index 000000000..19e410d85 --- /dev/null +++ b/src/zencore/include/zencore/mpscqueue.h @@ -0,0 +1,110 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <atomic> +#include <memory> +#include <new> +#include <optional> + +#ifdef __cpp_lib_hardware_interference_size +using std::hardware_constructive_interference_size; +using std::hardware_destructive_interference_size; +#else +// 64 bytes on x86-64 │ L1_CACHE_BYTES │ L1_CACHE_SHIFT │ __cacheline_aligned │ ... +constexpr std::size_t hardware_constructive_interference_size = 64; +constexpr std::size_t hardware_destructive_interference_size = 64; +#endif + +namespace zen { + +/** An untyped array of data with compile-time alignment and size derived from another type. */ +template<typename ElementType> +struct TypeCompatibleStorage +{ + ElementType* Data() { return (ElementType*)this; } + const ElementType* Data() const { return (const ElementType*)this; } + + alignas(ElementType) char DataMember; +}; + +/** Fast multi-producer/single-consumer unbounded concurrent queue. + + Based on http://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue + */ + +template<typename T> +class MpscQueue final +{ +public: + using ElementType = T; + + MpscQueue() + { + Node* Sentinel = new Node; + Head.store(Sentinel, std::memory_order_relaxed); + Tail = Sentinel; + } + + ~MpscQueue() + { + Node* Next = Tail->Next.load(std::memory_order_relaxed); + + // sentinel's value is already destroyed + delete Tail; + + while (Next != nullptr) + { + Tail = Next; + Next = Tail->Next.load(std::memory_order_relaxed); + + std::destroy_at((ElementType*)&Tail->Value); + delete Tail; + } + } + + template<typename... ArgTypes> + void Enqueue(ArgTypes&&... Args) + { + Node* New = new Node; + new (&New->Value) ElementType(std::forward<ArgTypes>(Args)...); + + Node* Prev = Head.exchange(New, std::memory_order_acq_rel); + Prev->Next.store(New, std::memory_order_release); + } + + std::optional<ElementType> Dequeue() + { + Node* Next = Tail->Next.load(std::memory_order_acquire); + + if (Next == nullptr) + { + return {}; + } + + ElementType* ValuePtr = (ElementType*)&Next->Value; + std::optional<ElementType> Res{std::move(*ValuePtr)}; + std::destroy_at(ValuePtr); + + delete Tail; // current sentinel + + Tail = Next; // new sentinel + return Res; + } + +private: + struct Node + { + std::atomic<Node*> Next{nullptr}; + TypeCompatibleStorage<ElementType> Value; + }; + +private: + std::atomic<Node*> Head; // accessed only by producers + alignas(hardware_constructive_interference_size) + Node* Tail; // accessed only by consumer, hence should be on a different cache line than `Head` +}; + +void mpscqueue_forcelink(); + +} // namespace zen diff --git a/src/zencore/include/zencore/refcount.h b/src/zencore/include/zencore/refcount.h new file mode 100644 index 000000000..f0bb6b85e --- /dev/null +++ b/src/zencore/include/zencore/refcount.h @@ -0,0 +1,186 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +#pragma once + +#include "atomic.h" +#include "zencore.h" + +#include <compare> + +namespace zen { + +/** + * Helper base class for reference counted objects using intrusive reference counts + * + * This class is pretty straightforward but does one thing which may be unexpected: + * + * - Instances on the stack are initialized with a reference count of one to ensure + * nobody tries to accidentally delete it. (TODO: is this really useful?) + */ +class RefCounted +{ +public: + RefCounted() = default; + virtual ~RefCounted() = default; + + inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); } + inline uint32_t Release() const + { + uint32_t refCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount); + if (refCount == 0) + { + delete this; + } + return refCount; + } + + // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it + + RefCounted(const RefCounted&) = delete; + RefCounted(RefCounted&&) = delete; + RefCounted& operator=(const RefCounted&) = delete; + RefCounted& operator=(RefCounted&&) = delete; + +protected: + inline uint32_t RefCount() const { return m_RefCount; } + +private: + uint32_t m_RefCount = 0; +}; + +/** + * Smart pointer for classes derived from RefCounted + */ + +template<class T> +class RefPtr +{ +public: + inline RefPtr() = default; + inline RefPtr(const RefPtr& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } + inline RefPtr(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } + inline ~RefPtr() { m_Ref && m_Ref->Release(); } + + [[nodiscard]] inline bool IsNull() const { return m_Ref == nullptr; } + inline explicit operator bool() const { return m_Ref != nullptr; } + inline operator T*() const { return m_Ref; } + inline T* operator->() const { return m_Ref; } + + inline std::strong_ordering operator<=>(const RefPtr& Rhs) const = default; + + inline RefPtr& operator=(T* Rhs) + { + Rhs && Rhs->AddRef(); + m_Ref && m_Ref->Release(); + m_Ref = Rhs; + return *this; + } + inline RefPtr& operator=(const RefPtr& Rhs) + { + if (&Rhs != this) + { + Rhs && Rhs->AddRef(); + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + } + return *this; + } + inline RefPtr& operator=(RefPtr&& Rhs) noexcept + { + if (&Rhs != this) + { + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + Rhs.m_Ref = nullptr; + } + return *this; + } + template<typename OtherType> + inline RefPtr& operator=(RefPtr<OtherType>&& Rhs) noexcept + { + if ((RefPtr*)&Rhs != this) + { + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + Rhs.m_Ref = nullptr; + } + return *this; + } + inline RefPtr(RefPtr&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } + template<typename OtherType> + explicit inline RefPtr(RefPtr<OtherType>&& Rhs) noexcept : m_Ref(Rhs.m_Ref) + { + Rhs.m_Ref = nullptr; + } + +private: + T* m_Ref = nullptr; + template<typename U> + friend class RefPtr; +}; + +/** + * Smart pointer for classes derived from RefCounted + * + * This variant does not decay to a raw pointer + * + */ + +template<class T> +class Ref +{ +public: + inline Ref() = default; + inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } + inline explicit Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } + inline ~Ref() { m_Ref && m_Ref->Release(); } + + template<typename DerivedType> + requires DerivedFrom<DerivedType, T> + inline Ref(const Ref<DerivedType>& Rhs) : Ref(Rhs.m_Ref) {} + + [[nodiscard]] inline bool IsNull() const { return m_Ref == nullptr; } + inline explicit operator bool() const { return m_Ref != nullptr; } + inline T* operator->() const { return m_Ref; } + inline T* Get() const { return m_Ref; } + + inline std::strong_ordering operator<=>(const Ref& Rhs) const = default; + + inline Ref& operator=(T* Rhs) + { + Rhs && Rhs->AddRef(); + m_Ref && m_Ref->Release(); + m_Ref = Rhs; + return *this; + } + inline Ref& operator=(const Ref& Rhs) + { + if (&Rhs != this) + { + Rhs && Rhs->AddRef(); + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + } + return *this; + } + inline Ref& operator=(Ref&& Rhs) noexcept + { + if (&Rhs != this) + { + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + Rhs.m_Ref = nullptr; + } + return *this; + } + inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } + +private: + T* m_Ref = nullptr; + + template<class U> + friend class Ref; +}; + +void refcount_forcelink(); + +} // namespace zen diff --git a/src/zencore/include/zencore/scopeguard.h b/src/zencore/include/zencore/scopeguard.h new file mode 100644 index 000000000..d04c8ed9c --- /dev/null +++ b/src/zencore/include/zencore/scopeguard.h @@ -0,0 +1,45 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <type_traits> +#include "logging.h" +#include "zencore.h" + +namespace zen { + +template<typename T> +class [[nodiscard]] ScopeGuardImpl +{ +public: + inline ScopeGuardImpl(T&& func) : m_guardFunc(func) {} + ~ScopeGuardImpl() + { + if (!m_dismissed) + { + try + { + m_guardFunc(); + } + catch (std::exception& Ex) + { + ZEN_ERROR("scope guard threw exception: '{}'", Ex.what()); + } + } + } + + void Dismiss() { m_dismissed = true; } + +private: + bool m_dismissed = false; + T m_guardFunc; +}; + +template<typename T> +ScopeGuardImpl<T> +MakeGuard(T&& fn) +{ + return ScopeGuardImpl<T>(std::move(fn)); +} + +} // namespace zen diff --git a/src/zencore/include/zencore/session.h b/src/zencore/include/zencore/session.h new file mode 100644 index 000000000..dd90197bf --- /dev/null +++ b/src/zencore/include/zencore/session.h @@ -0,0 +1,14 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +namespace zen { + +struct Oid; + +ZENCORE_API [[nodiscard]] Oid GetSessionId(); +ZENCORE_API [[nodiscard]] std::string_view GetSessionIdString(); + +} // namespace zen diff --git a/src/zencore/include/zencore/sha1.h b/src/zencore/include/zencore/sha1.h new file mode 100644 index 000000000..fc26f442b --- /dev/null +++ b/src/zencore/include/zencore/sha1.h @@ -0,0 +1,76 @@ +// ////////////////////////////////////////////////////////// +// sha1.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +#include <stdint.h> +#include <compare> +#include "zencore.h" + +namespace zen { + +class StringBuilderBase; + +struct SHA1 +{ + uint8_t Hash[20]; + + inline auto operator<=>(const SHA1& rhs) const = default; + + static const int StringLength = 40; + typedef char String_t[StringLength + 1]; + + static SHA1 HashMemory(const void* data, size_t byteCount); + static SHA1 FromHexString(const char* string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static SHA1 Zero; // Initialized to all zeroes +}; + +/** + * Utility class for computing SHA1 hashes + */ +class SHA1Stream +{ +public: + SHA1Stream(); + + /** compute SHA1 of a memory block + + \note SHA1 class contains a slightly more convenient helper function for this use case + \see SHA1::fromMemory() + */ + SHA1 Compute(const void* data, size_t byteCount); + + /// Begin streaming SHA1 compute (not needed on freshly constructed SHA1Stream instance) + void Reset(); + /// Append another chunk + SHA1Stream& Append(const void* data, size_t byteCount); + /// Obtain final SHA1 hash. If you wish to reuse the SHA1Stream instance call reset() + SHA1 GetHash(); + +private: + void ProcessBlock(const void* data); + void ProcessBuffer(); + + enum + { + /// split into 64 byte blocks (=> 512 bits) + BlockSize = 512 / 8, + HashBytes = 20, + HashValues = HashBytes / 4 + }; + + uint64_t m_NumBytes; // size of processed data in bytes + size_t m_BufferSize; // valid bytes in m_buffer + uint8_t m_Buffer[BlockSize]; // bytes not processed yet + uint32_t m_Hash[HashValues]; +}; + +void sha1_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/sharedbuffer.h b/src/zencore/include/zencore/sharedbuffer.h new file mode 100644 index 000000000..97c5a9d21 --- /dev/null +++ b/src/zencore/include/zencore/sharedbuffer.h @@ -0,0 +1,167 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/iobuffer.h> +#include <zencore/memory.h> +#include <zencore/refcount.h> + +#include <memory.h> + +namespace zen { + +class SharedBuffer; + +/** + * Reference to a memory buffer with a single owner + * + * Internally + */ +class UniqueBuffer +{ +public: + UniqueBuffer() = default; + UniqueBuffer(UniqueBuffer&&) = default; + UniqueBuffer& operator=(UniqueBuffer&&) = default; + UniqueBuffer(const UniqueBuffer&) = delete; + UniqueBuffer& operator=(const UniqueBuffer&) = delete; + + ZENCORE_API explicit UniqueBuffer(IoBufferCore* Owner); + + [[nodiscard]] void* GetData() { return m_Buffer ? m_Buffer->MutableDataPointer() : nullptr; } + [[nodiscard]] const void* GetData() const { return m_Buffer ? m_Buffer->DataPointer() : nullptr; } + [[nodiscard]] size_t GetSize() const { return m_Buffer ? m_Buffer->DataBytes() : 0; } + + operator MutableMemoryView() { return GetMutableView(); } + operator MemoryView() const { return GetView(); } + + /** + * Returns true if this does not point to a buffer owner. + * + * A null buffer is always owned, materialized, and empty. + */ + [[nodiscard]] inline bool IsNull() const { return m_Buffer.IsNull(); } + + /** Reset this to null. */ + ZENCORE_API void Reset(); + + [[nodiscard]] inline MutableMemoryView GetMutableView() { return MutableMemoryView(GetData(), GetSize()); } + [[nodiscard]] inline MemoryView GetView() const { return MemoryView(GetData(), GetSize()); } + + /** Make an uninitialized owned buffer of the specified size. */ + [[nodiscard]] ZENCORE_API static UniqueBuffer Alloc(uint64_t Size); + + /** Make a non-owned view of the input. */ + [[nodiscard]] ZENCORE_API static UniqueBuffer MakeMutableView(void* DataPtr, uint64_t Size); + + /** + * Convert this to an immutable shared buffer, leaving this null. + * + * Steals the buffer owner from the unique buffer. + */ + [[nodiscard]] ZENCORE_API SharedBuffer MoveToShared(); + +private: + // This may be null, for a default constructed UniqueBuffer only + RefPtr<IoBufferCore> m_Buffer; + + friend class SharedBuffer; +}; + +/** + * Reference to a memory buffer with shared ownership + */ +class SharedBuffer +{ +public: + SharedBuffer() = default; + ZENCORE_API explicit SharedBuffer(UniqueBuffer&&); + inline explicit SharedBuffer(IoBufferCore* Owner) : m_Buffer(Owner) {} + ZENCORE_API explicit SharedBuffer(IoBuffer&& Buffer) : m_Buffer(std::move(Buffer.m_Core)) {} + ZENCORE_API explicit SharedBuffer(const IoBuffer& Buffer) : m_Buffer(Buffer.m_Core) {} + ZENCORE_API explicit SharedBuffer(RefPtr<IoBufferCore>&& Owner) : m_Buffer(std::move(Owner)) {} + + [[nodiscard]] const void* GetData() const + { + if (m_Buffer) + { + return m_Buffer->DataPointer(); + } + return nullptr; + } + + [[nodiscard]] size_t GetSize() const + { + if (m_Buffer) + { + return m_Buffer->DataBytes(); + } + return 0; + } + + inline void MakeImmutable() + { + ZEN_ASSERT(m_Buffer); + m_Buffer->SetIsImmutable(true); + } + + /** Returns a buffer that is owned, by cloning if not owned. */ + [[nodiscard]] ZENCORE_API SharedBuffer MakeOwned() const&; + [[nodiscard]] ZENCORE_API SharedBuffer MakeOwned() &&; + + [[nodiscard]] bool IsOwned() const { return !m_Buffer || m_Buffer->IsOwned(); } + [[nodiscard]] inline bool IsNull() const { return !m_Buffer; } + inline void Reset() { m_Buffer = nullptr; } + + [[nodiscard]] MemoryView GetView() const + { + if (m_Buffer) + { + return MemoryView(m_Buffer->DataPointer(), m_Buffer->DataBytes()); + } + else + { + return MemoryView(); + } + } + operator MemoryView() const { return GetView(); } + + /** Returns true if this points to a buffer owner. */ + [[nodiscard]] inline explicit operator bool() const { return !IsNull(); } + + [[nodiscard]] inline IoBuffer AsIoBuffer() const { return IoBuffer(m_Buffer); } + + SharedBuffer& operator=(UniqueBuffer&& Rhs) + { + m_Buffer = std::move(Rhs.m_Buffer); + return *this; + } + + std::strong_ordering operator<=>(const SharedBuffer& Rhs) const = default; + + /** Make a non-owned view of the input */ + [[nodiscard]] inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); } + /** Make a non-owning view of the memory of the contiguous container. */ + [[nodiscard]] inline static SharedBuffer MakeView(const ContiguousRange auto& Container) + { + std::span Span = Container; + return MakeView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type)); + } + /** Make a non-owned view of the input */ + [[nodiscard]] ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size); + /** Make a non-owned view of the input */ + [[nodiscard]] ZENCORE_API static SharedBuffer MakeView(MemoryView View, SharedBuffer OuterBuffer); + /** Make an owned clone of the buffer */ + [[nodiscard]] ZENCORE_API SharedBuffer Clone(); + /** Make an owned clone of the memory in the input view */ + [[nodiscard]] ZENCORE_API static SharedBuffer Clone(MemoryView View); + +private: + RefPtr<IoBufferCore> m_Buffer; +}; + +void sharedbuffer_forcelink(); + +} // namespace zen diff --git a/src/zencore/include/zencore/stats.h b/src/zencore/include/zencore/stats.h new file mode 100644 index 000000000..1a0817b99 --- /dev/null +++ b/src/zencore/include/zencore/stats.h @@ -0,0 +1,295 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <atomic> +#include <random> + +namespace zen { +class CbObjectWriter; +} + +namespace zen::metrics { + +template<typename T> +class Gauge +{ +public: + Gauge() : m_Value{0} {} + + T Value() const { return m_Value; } + void SetValue(T Value) { m_Value = Value; } + +private: + std::atomic<T> m_Value; +}; + +/** Stats counter + * + * A counter is modified by adding or subtracting a value from a current value. + * This would typically be used to track number of requests in flight, number + * of active jobs etc + * + */ +class Counter +{ +public: + inline void SetValue(uint64_t Value) { m_count = Value; } + inline uint64_t Value() const { return m_count; } + + inline void Increment(int64_t AddValue) { m_count.fetch_add(AddValue); } + inline void Decrement(int64_t SubValue) { m_count.fetch_sub(SubValue); } + inline void Clear() { m_count.store(0, std::memory_order_release); } + +private: + std::atomic<uint64_t> m_count{0}; +}; + +/** Exponential Weighted Moving Average + + This is very raw, to use as little state as possible. If we + want to use this more broadly in user code we should perhaps + add a more user-friendly wrapper + */ + +class RawEWMA +{ +public: + /// <summary> + /// Update EWMA with new measure + /// </summary> + /// <param name="Alpha">Smoothing factor (between 0 and 1)</param> + /// <param name="Interval">Elapsed time since last</param> + /// <param name="Count">Value</param> + /// <param name="IsInitialUpdate">Whether this is the first update or not</param> + void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate); + double Rate() const; + +private: + std::atomic<double> m_Rate = 0; +}; + +/// <summary> +/// Tracks rate of events over time (i.e requests/sec), using +/// exponential moving averages +/// </summary> +class Meter +{ +public: + Meter(); + ~Meter(); + + inline uint64_t Count() const { return m_TotalCount; } + double Rate1(); // One-minute rate + double Rate5(); // Five-minute rate + double Rate15(); // Fifteen-minute rate + double MeanRate() const; // Mean rate since instantiation of this meter + void Mark(uint64_t Count = 1); // Register one or more events + +private: + std::atomic<uint64_t> m_TotalCount{0}; // Accumulator counting number of marks since beginning + std::atomic<uint64_t> m_PendingCount{0}; // Pending EWMA update accumulator + std::atomic<uint64_t> m_StartTick{0}; // Time this was instantiated (for mean) + std::atomic<uint64_t> m_LastTick{0}; // Timestamp of last EWMA tick + std::atomic<int64_t> m_Remainder{0}; // Tracks the "modulo" of tick time + bool m_IsFirstTick = true; + RawEWMA m_RateM1; + RawEWMA m_RateM5; + RawEWMA m_RateM15; + + void TickIfNecessary(); + void Tick(); +}; + +/** Moment-in-time snapshot of a distribution + */ +class SampleSnapshot +{ +public: + SampleSnapshot(std::vector<double>&& Values); + ~SampleSnapshot(); + + uint32_t Size() const { return (uint32_t)m_Values.size(); } + double GetQuantileValue(double Quantile); + double GetMedian() { return GetQuantileValue(0.5); } + double Get75Percentile() { return GetQuantileValue(0.75); } + double Get95Percentile() { return GetQuantileValue(0.95); } + double Get98Percentile() { return GetQuantileValue(0.98); } + double Get99Percentile() { return GetQuantileValue(0.99); } + double Get999Percentile() { return GetQuantileValue(0.999); } + const std::vector<double>& GetValues() const; + +private: + std::vector<double> m_Values; +}; + +/** Randomly selects samples from a stream. Uses Vitter's + Algorithm R to produce a statistically representative sample. + + http://www.cs.umd.edu/~samir/498/vitter.pdf - Random Sampling with a Reservoir + */ + +class UniformSample +{ +public: + UniformSample(uint32_t ReservoirSize); + ~UniformSample(); + + void Clear(); + uint32_t Size() const; + void Update(int64_t Value); + SampleSnapshot Snapshot() const; + + template<Invocable<int64_t> T> + void IterateValues(T Callback) const + { + for (const auto& Value : m_Values) + { + Callback(Value); + } + } + +private: + std::atomic<uint64_t> m_SampleCounter{0}; + std::vector<std::atomic<int64_t>> m_Values; +}; + +/** Track (probabilistic) sample distribution along with min/max + */ +class Histogram +{ +public: + Histogram(int32_t SampleCount = 1028); + ~Histogram(); + + void Clear(); + void Update(int64_t Value); + int64_t Max() const; + int64_t Min() const; + double Mean() const; + uint64_t Count() const; + SampleSnapshot Snapshot() const { return m_Sample.Snapshot(); } + +private: + UniformSample m_Sample; + std::atomic<int64_t> m_Min{0}; + std::atomic<int64_t> m_Max{0}; + std::atomic<int64_t> m_Sum{0}; + std::atomic<int64_t> m_Count{0}; +}; + +/** Track timing and frequency of some operation + + Example usage would be to track frequency and duration of network + requests, or function calls. + + */ +class OperationTiming +{ +public: + OperationTiming(int32_t SampleCount = 514); + ~OperationTiming(); + + void Update(int64_t Duration); + int64_t Max() const; + int64_t Min() const; + double Mean() const; + uint64_t Count() const; + SampleSnapshot Snapshot() const { return m_Histogram.Snapshot(); } + + double Rate1() { return m_Meter.Rate1(); } + double Rate5() { return m_Meter.Rate5(); } + double Rate15() { return m_Meter.Rate15(); } + double MeanRate() const { return m_Meter.MeanRate(); } + + struct Scope + { + Scope(OperationTiming& Outer); + ~Scope(); + + void Stop(); + void Cancel(); + + private: + OperationTiming& m_Outer; + uint64_t m_StartTick; + }; + +private: + Meter m_Meter; + Histogram m_Histogram; +}; + +/** Metrics for network requests + + Aggregates tracking of duration, payload sizes into a single + class + + */ +class RequestStats +{ +public: + RequestStats(int32_t SampleCount = 514); + ~RequestStats(); + + void Update(int64_t Duration, int64_t Bytes); + uint64_t Count() const; + + // Timing + + int64_t MaxDuration() const { return m_BytesHistogram.Max(); } + int64_t MinDuration() const { return m_BytesHistogram.Min(); } + double MeanDuration() const { return m_BytesHistogram.Mean(); } + SampleSnapshot DurationSnapshot() const { return m_RequestTimeHistogram.Snapshot(); } + double Rate1() { return m_RequestMeter.Rate1(); } + double Rate5() { return m_RequestMeter.Rate5(); } + double Rate15() { return m_RequestMeter.Rate15(); } + double MeanRate() const { return m_RequestMeter.MeanRate(); } + + // Bytes + + int64_t MaxBytes() const { return m_BytesHistogram.Max(); } + int64_t MinBytes() const { return m_BytesHistogram.Min(); } + double MeanBytes() const { return m_BytesHistogram.Mean(); } + SampleSnapshot BytesSnapshot() const { return m_BytesHistogram.Snapshot(); } + double ByteRate1() { return m_BytesMeter.Rate1(); } + double ByteRate5() { return m_BytesMeter.Rate5(); } + double ByteRate15() { return m_BytesMeter.Rate15(); } + double ByteMeanRate() const { return m_BytesMeter.MeanRate(); } + + struct Scope + { + Scope(OperationTiming& Outer); + ~Scope(); + + void Cancel(); + + private: + OperationTiming& m_Outer; + uint64_t m_StartTick; + }; + + void EmitSnapshot(std::string_view Tag, CbObjectWriter& Cbo); + +private: + Meter m_RequestMeter; + Meter m_BytesMeter; + Histogram m_RequestTimeHistogram; + Histogram m_BytesHistogram; +}; + +void EmitSnapshot(std::string_view Tag, OperationTiming& Stat, CbObjectWriter& Cbo); +void EmitSnapshot(std::string_view Tag, const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor); +void EmitSnapshot(std::string_view Tag, Meter& Stat, CbObjectWriter& Cbo); + +void EmitSnapshot(const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor); + +} // namespace zen::metrics + +namespace zen { + +extern void stats_forcelink(); + +} // namespace zen diff --git a/src/zencore/include/zencore/stream.h b/src/zencore/include/zencore/stream.h new file mode 100644 index 000000000..9e4996249 --- /dev/null +++ b/src/zencore/include/zencore/stream.h @@ -0,0 +1,90 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/memory.h> +#include <zencore/thread.h> + +#include <vector> + +namespace zen { + +/** + * Binary stream writer + */ + +class BinaryWriter +{ +public: + inline BinaryWriter() = default; + ~BinaryWriter() = default; + + inline void Write(const void* DataPtr, size_t ByteCount) + { + Write(DataPtr, ByteCount, m_Offset); + m_Offset += ByteCount; + } + + inline void Write(MemoryView Memory) { Write(Memory.GetData(), Memory.GetSize()); } + void Write(std::initializer_list<const MemoryView> Buffers); + + inline uint64_t CurrentOffset() const { return m_Offset; } + + inline const uint8_t* Data() const { return m_Buffer.data(); } + inline const uint8_t* GetData() const { return m_Buffer.data(); } + inline uint64_t Size() const { return m_Buffer.size(); } + inline uint64_t GetSize() const { return m_Buffer.size(); } + void Reset(); + + inline MemoryView GetView() const { return MemoryView(m_Buffer.data(), m_Offset); } + inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Buffer.data(), m_Offset); } + +private: + std::vector<uint8_t> m_Buffer; + uint64_t m_Offset = 0; + + void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset); +}; + +inline MemoryView +MakeMemoryView(const BinaryWriter& Stream) +{ + return MemoryView(Stream.Data(), Stream.Size()); +} + +/** + * Binary stream reader + */ + +class BinaryReader +{ +public: + inline BinaryReader(const void* Buffer, uint64_t Size) : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer)), m_BufferSize(Size) {} + inline BinaryReader(MemoryView Buffer) + : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer.GetData())) + , m_BufferSize(Buffer.GetSize()) + { + } + + inline void Read(void* DataPtr, size_t ByteCount) + { + memcpy(DataPtr, m_BufferBase + m_Offset, ByteCount); + m_Offset += ByteCount; + } + + inline uint64_t Size() const { return m_BufferSize; } + inline uint64_t GetSize() const { return Size(); } + inline uint64_t CurrentOffset() const { return m_Offset; } + inline void Skip(size_t ByteCount) { m_Offset += ByteCount; }; + +private: + const uint8_t* m_BufferBase; + uint64_t m_BufferSize; + uint64_t m_Offset = 0; +}; + +void stream_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h new file mode 100644 index 000000000..ab111ff81 --- /dev/null +++ b/src/zencore/include/zencore/string.h @@ -0,0 +1,1115 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "intmath.h" +#include "zencore.h" + +#include <stdint.h> +#include <string.h> +#include <charconv> +#include <codecvt> +#include <compare> +#include <concepts> +#include <optional> +#include <span> +#include <string_view> + +#include <type_traits> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +inline bool +StringEquals(const char8_t* s1, const char* s2) +{ + return strcmp(reinterpret_cast<const char*>(s1), s2) == 0; +} + +inline bool +StringEquals(const char* s1, const char* s2) +{ + return strcmp(s1, s2) == 0; +} + +inline size_t +StringLength(const char* str) +{ + return strlen(str); +} + +inline bool +StringEquals(const wchar_t* s1, const wchar_t* s2) +{ + return wcscmp(s1, s2) == 0; +} + +inline size_t +StringLength(const wchar_t* str) +{ + return wcslen(str); +} + +////////////////////////////////////////////////////////////////////////// +// File name helpers +// + +ZENCORE_API const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr); + +////////////////////////////////////////////////////////////////////////// +// Text formatting of numbers +// + +ZENCORE_API bool ToString(std::span<char> Buffer, uint64_t Num); +ZENCORE_API bool ToString(std::span<char> Buffer, int64_t Num); + +struct TextNumBase +{ + inline const char* c_str() const { return m_Buffer; } + inline operator std::string_view() const { return std::string_view(m_Buffer); } + +protected: + char m_Buffer[24]; +}; + +struct IntNum : public TextNumBase +{ + inline IntNum(UnsignedIntegral auto Number) { ToString(m_Buffer, uint64_t(Number)); } + inline IntNum(SignedIntegral auto Number) { ToString(m_Buffer, int64_t(Number)); } +}; + +////////////////////////////////////////////////////////////////////////// +// +// Quick-and-dirty string builder. Good enough for me, but contains traps +// and not-quite-ideal behaviour especially when mixing character types etc +// + +template<typename C> +class StringBuilderImpl +{ +public: + StringBuilderImpl() = default; + ZENCORE_API ~StringBuilderImpl(); + + StringBuilderImpl(const StringBuilderImpl&) = delete; + StringBuilderImpl(const StringBuilderImpl&&) = delete; + const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete; + const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete; + + inline size_t AddUninitialized(size_t Count) + { + EnsureCapacity(Count); + const size_t OldCount = Size(); + m_CurPos += Count; + return OldCount; + } + + StringBuilderImpl& Append(C OneChar) + { + EnsureCapacity(1); + + *m_CurPos++ = OneChar; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const std::string_view& String) + { + const size_t len = String.size(); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = String[i]; + + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const std::u8string_view& String) + { + const size_t len = String.size(); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = String[i]; + + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const char* NulTerminatedString) + { + size_t StringLen = StringLength(NulTerminatedString); + + return AppendAscii({NulTerminatedString, StringLen}); + } + + inline StringBuilderImpl& Append(const char8_t* NulTerminatedString) + { + // This is super hacky and not fully functional - needs better + // solution + if constexpr (sizeof(C) == 1) + { + size_t len = StringLength((const char*)NulTerminatedString); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = C(NulTerminatedString[i]); + + m_CurPos += len; + } + else + { + ZEN_NOT_IMPLEMENTED(); + } + + return *this; + } + + inline StringBuilderImpl& AppendAsciiRange(const char* BeginString, const char* EndString) + { + EnsureCapacity(EndString - BeginString); + + while (BeginString != EndString) + *m_CurPos++ = *BeginString++; + + return *this; + } + + inline StringBuilderImpl& Append(const C* NulTerminatedString) + { + size_t Len = StringLength(NulTerminatedString); + + EnsureCapacity(Len); + memcpy(m_CurPos, NulTerminatedString, Len * sizeof(C)); + m_CurPos += Len; + + return *this; + } + + inline StringBuilderImpl& Append(const C* NulTerminatedString, size_t MaxChars) + { + size_t len = Min(MaxChars, StringLength(NulTerminatedString)); + + EnsureCapacity(len); + memcpy(m_CurPos, NulTerminatedString, len * sizeof(C)); + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendRange(const C* BeginString, const C* EndString) + { + size_t Len = EndString - BeginString; + + EnsureCapacity(Len); + memcpy(m_CurPos, BeginString, Len * sizeof(C)); + m_CurPos += Len; + + return *this; + } + + inline StringBuilderImpl& Append(const std::basic_string_view<C>& String) + { + return AppendRange(String.data(), String.data() + String.size()); + } + + inline StringBuilderImpl& AppendBool(bool v) + { + // This is a method instead of a << operator overload as the latter can + // easily get called with non-bool types like pointers. It is a very + // subtle behaviour that can cause bugs. + using namespace std::literals; + if (v) + { + return AppendAscii("true"sv); + } + return AppendAscii("false"sv); + } + + inline void RemoveSuffix(uint32_t Count) + { + ZEN_ASSERT(Count <= Size()); + m_CurPos -= Count; + } + + inline const C* c_str() const + { + EnsureNulTerminated(); + return m_Base; + } + + inline C* Data() + { + EnsureNulTerminated(); + return m_Base; + } + + inline const C* Data() const + { + EnsureNulTerminated(); + return m_Base; + } + + inline size_t Size() const { return m_CurPos - m_Base; } + inline bool IsDynamic() const { return m_IsDynamic; } + inline void Reset() { m_CurPos = m_Base; } + + inline StringBuilderImpl& operator<<(uint64_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int64_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint32_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int32_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint16_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int16_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint8_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int8_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + + inline StringBuilderImpl& operator<<(const char* str) { return AppendAscii(str); } + inline StringBuilderImpl& operator<<(const std::string_view str) { return AppendAscii(str); } + inline StringBuilderImpl& operator<<(const std::u8string_view str) { return AppendAscii(str); } + +protected: + inline void Init(C* Base, size_t Capacity) + { + m_Base = m_CurPos = Base; + m_End = Base + Capacity; + } + + inline void EnsureNulTerminated() const { *m_CurPos = '\0'; } + + inline void EnsureCapacity(size_t ExtraRequired) + { + // precondition: we know the current buffer has enough capacity + // for the existing string including NUL terminator + + if ((m_CurPos + ExtraRequired) < m_End) + return; + + Extend(ExtraRequired); + } + + ZENCORE_API void Extend(size_t ExtraCapacity); + ZENCORE_API void* AllocBuffer(size_t ByteCount); + ZENCORE_API void FreeBuffer(void* Buffer, size_t ByteCount); + + ZENCORE_API [[noreturn]] void Fail(const char* FailReason); // note: throws exception + + C* m_Base; + C* m_CurPos; + C* m_End; + bool m_IsDynamic = false; + bool m_IsExtendable = false; +}; + +////////////////////////////////////////////////////////////////////////// + +extern template class StringBuilderImpl<char>; + +inline StringBuilderImpl<char>& +operator<<(StringBuilderImpl<char>& Builder, char Char) +{ + return Builder.Append(Char); +} + +class StringBuilderBase : public StringBuilderImpl<char> +{ +public: + inline StringBuilderBase(char* bufferPointer, size_t bufferCapacity) { Init(bufferPointer, bufferCapacity); } + inline ~StringBuilderBase() = default; + + // Note that we don't need a terminator for the string_view so we avoid calling data() here + inline operator std::string_view() const { return std::string_view(m_Base, m_CurPos - m_Base); } + inline std::string_view ToView() const { return std::string_view(m_Base, m_CurPos - m_Base); } + inline std::string ToString() const { return std::string{Data(), Size()}; } + + inline void AppendCodepoint(uint32_t cp) + { + if (cp < 0x80) // one octet + { + Append(static_cast<char8_t>(cp)); + } + else if (cp < 0x800) + { + EnsureCapacity(2); // two octets + m_CurPos[0] = static_cast<char8_t>((cp >> 6) | 0xc0); + m_CurPos[1] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 2; + } + else if (cp < 0x10000) + { + EnsureCapacity(3); // three octets + m_CurPos[0] = static_cast<char8_t>((cp >> 12) | 0xe0); + m_CurPos[1] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80); + m_CurPos[2] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 3; + } + else + { + EnsureCapacity(4); // four octets + m_CurPos[0] = static_cast<char8_t>((cp >> 18) | 0xf0); + m_CurPos[1] = static_cast<char8_t>(((cp >> 12) & 0x3f) | 0x80); + m_CurPos[2] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80); + m_CurPos[3] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 4; + } + } +}; + +template<size_t N> +class StringBuilder : public StringBuilderBase +{ +public: + inline StringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) {} + inline ~StringBuilder() = default; + +private: + char m_StringBuffer[N]; +}; + +template<size_t N> +class ExtendableStringBuilder : public StringBuilderBase +{ +public: + inline ExtendableStringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) { m_IsExtendable = true; } + inline ~ExtendableStringBuilder() = default; + +private: + char m_StringBuffer[N]; +}; + +template<size_t N> +class WriteToString : public ExtendableStringBuilder<N> +{ +public: + template<typename... ArgTypes> + explicit WriteToString(ArgTypes&&... Args) + { + (*this << ... << std::forward<ArgTypes>(Args)); + } +}; + +////////////////////////////////////////////////////////////////////////// + +extern template class StringBuilderImpl<wchar_t>; + +class WideStringBuilderBase : public StringBuilderImpl<wchar_t> +{ +public: + inline WideStringBuilderBase(wchar_t* BufferPointer, size_t BufferCapacity) { Init(BufferPointer, BufferCapacity); } + inline ~WideStringBuilderBase() = default; + + inline operator std::wstring_view() const { return std::wstring_view{Data(), Size()}; } + inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; } + inline std::wstring ToString() const { return std::wstring{Data(), Size()}; } + + inline StringBuilderImpl& operator<<(const std::wstring_view str) { return Append((const wchar_t*)str.data(), str.size()); } + inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); } + using StringBuilderImpl:: operator<<; +}; + +template<size_t N> +class WideStringBuilder : public WideStringBuilderBase +{ +public: + inline WideStringBuilder() : WideStringBuilderBase(m_Buffer, N) {} + ~WideStringBuilder() = default; + +private: + wchar_t m_Buffer[N]; +}; + +template<size_t N> +class ExtendableWideStringBuilder : public WideStringBuilderBase +{ +public: + inline ExtendableWideStringBuilder() : WideStringBuilderBase(m_Buffer, N) { m_IsExtendable = true; } + ~ExtendableWideStringBuilder() = default; + +private: + wchar_t m_Buffer[N]; +}; + +template<size_t N> +class WriteToWideString : public ExtendableWideStringBuilder<N> +{ +public: + template<typename... ArgTypes> + explicit WriteToWideString(ArgTypes&&... Args) + { + (*this << ... << Forward<ArgTypes>(Args)); + } +}; + +////////////////////////////////////////////////////////////////////////// + +void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out); +void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out); +void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out); +std::wstring Utf8ToWide(const std::string_view& wstr); + +void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out); +std::string WideToUtf8(const wchar_t* wstr); +void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out); +std::string WideToUtf8(const std::wstring_view Wstr); + +inline uint8_t +Char2Nibble(char c) +{ + if (c >= '0' && c <= '9') + { + return uint8_t(c - '0'); + } + if (c >= 'a' && c <= 'f') + { + return uint8_t(c - 'a' + 10); + } + if (c >= 'A' && c <= 'F') + { + return uint8_t(c - 'A' + 10); + } + return uint8_t(0xff); +}; + +static constexpr const char HexChars[] = "0123456789abcdef"; + +/// <summary> +/// Parse hex string into a byte buffer +/// </summary> +/// <param name="string">Input string</param> +/// <param name="characterCount">Number of characters in string</param> +/// <param name="outPtr">Pointer to output buffer</param> +/// <returns>true if the input consisted of all valid hexadecimal characters</returns> + +inline bool +ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr) +{ + ZEN_ASSERT((CharacterCount & 1) == 0); + + uint8_t allBits = 0; + + while (CharacterCount) + { + uint8_t n0 = Char2Nibble(InputString[0]); + uint8_t n1 = Char2Nibble(InputString[1]); + + allBits |= n0 | n1; + + *OutPtr = (n0 << 4) | n1; + + OutPtr += 1; + InputString += 2; + CharacterCount -= 2; + } + + return (allBits & 0x80) == 0; +} + +inline void +ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString) +{ + while (ByteCount--) + { + uint8_t byte = *InputData++; + + *OutString++ = HexChars[byte >> 4]; + *OutString++ = HexChars[byte & 15]; + } +} + +inline bool +ParseHexNumber(const char* InputString, size_t CharacterCount, uint8_t* OutPtr) +{ + ZEN_ASSERT((CharacterCount & 1) == 0); + + uint8_t allBits = 0; + + InputString += CharacterCount; + while (CharacterCount) + { + InputString -= 2; + uint8_t n0 = Char2Nibble(InputString[0]); + uint8_t n1 = Char2Nibble(InputString[1]); + + allBits |= n0 | n1; + + *OutPtr = (n0 << 4) | n1; + + OutPtr += 1; + CharacterCount -= 2; + } + + return (allBits & 0x80) == 0; +} + +inline void +ToHexNumber(const uint8_t* InputData, size_t ByteCount, char* OutString) +{ + InputData += ByteCount; + while (ByteCount--) + { + uint8_t byte = *(--InputData); + + *OutString++ = HexChars[byte >> 4]; + *OutString++ = HexChars[byte & 15]; + } +} + +/// <summary> +/// Generates a hex number from a pointer to an integer type, this formats the number in the correct order for a hexadecimal number +/// </summary> +/// <param name="Value">Integer value type</param> +/// <param name="outString">Output buffer where resulting string is written</param> +void +ToHexNumber(UnsignedIntegral auto Value, char* OutString) +{ + ToHexNumber((const uint8_t*)&Value, sizeof(Value), OutString); + OutString[sizeof(Value) * 2] = 0; +} + +/// <summary> +/// Parse hex number string into a value, this formats the number in the correct order for a hexadecimal number +/// </summary> +/// <param name="string">Input string</param> +/// <param name="characterCount">Number of characters in string</param> +/// <param name="OutValue">Pointer to output value</param> +/// <returns>true if the input consisted of all valid hexadecimal characters</returns> +bool +ParseHexNumber(const std::string HexString, UnsignedIntegral auto& OutValue) +{ + return ParseHexNumber(HexString.c_str(), sizeof(OutValue) * 2, (uint8_t*)&OutValue); +} + +////////////////////////////////////////////////////////////////////////// +// Format numbers for humans +// + +ZENCORE_API size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer); +ZENCORE_API size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer); +ZENCORE_API size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer); +ZENCORE_API size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer); +ZENCORE_API size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer); + +struct NiceBase +{ + inline const char* c_str() const { return m_Buffer; } + inline operator std::string_view() const { return std::string_view(m_Buffer); } + +protected: + char m_Buffer[16]; +}; + +struct NiceNum : public NiceBase +{ + inline NiceNum(uint64_t Num) { NiceNumToBuffer(Num, m_Buffer); } +}; + +struct NiceBytes : public NiceBase +{ + inline NiceBytes(uint64_t Num) { NiceBytesToBuffer(Num, m_Buffer); } +}; + +struct NiceByteRate : public NiceBase +{ + inline NiceByteRate(uint64_t Bytes, uint64_t TimeMilliseconds) { NiceByteRateToBuffer(Bytes, TimeMilliseconds, m_Buffer); } +}; + +struct NiceLatencyNs : public NiceBase +{ + inline NiceLatencyNs(uint64_t Milliseconds) { NiceLatencyNsToBuffer(Milliseconds, m_Buffer); } +}; + +struct NiceTimeSpanMs : public NiceBase +{ + inline NiceTimeSpanMs(uint64_t Milliseconds) { NiceTimeSpanMsToBuffer(Milliseconds, m_Buffer); } +}; + +////////////////////////////////////////////////////////////////////////// + +inline std::string +NiceRate(uint64_t Num, uint32_t DurationMilliseconds, const char* Unit = "B") +{ + char Buffer[32]; + + if (DurationMilliseconds) + { + // Leave a little of 'Buffer' for the "Unit/s" suffix + std::span<char> BufferSpan(Buffer, sizeof(Buffer) - 8); + NiceNumToBuffer(Num * 1000 / DurationMilliseconds, BufferSpan); + } + else + { + strcpy(Buffer, "0"); + } + + strncat(Buffer, Unit, 4); + strcat(Buffer, "/s"); + + return Buffer; +} + +////////////////////////////////////////////////////////////////////////// + +template<Integral T> +std::optional<T> +ParseInt(const std::string_view& Input) +{ + T Out = 0; + const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out); + if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range) + { + return std::nullopt; + } + return Out; +} + +////////////////////////////////////////////////////////////////////////// + +constexpr uint32_t +HashStringDjb2(const std::string_view& InString) +{ + uint32_t HashValue = 5381; + + for (int CurChar : InString) + { + HashValue = HashValue * 33 + CurChar; + } + + return HashValue; +} + +constexpr uint32_t +HashStringAsLowerDjb2(const std::string_view& InString) +{ + uint32_t HashValue = 5381; + + for (uint8_t CurChar : InString) + { + CurChar -= ((CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic + HashValue = HashValue * 33 + CurChar; + } + + return HashValue; +} + +////////////////////////////////////////////////////////////////////////// + +inline std::string +ToLower(const std::string_view& InString) +{ + std::string Out(InString); + + for (char& CurChar : Out) + { + CurChar -= (uint8_t(CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic + } + + return Out; +} + +////////////////////////////////////////////////////////////////////////// + +template<typename Fn> +uint32_t +ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func) +{ + const char* It = Str.data(); + const char* End = It + Str.length(); + uint32_t Count = 0; + + while (It != End) + { + if (*It == Delim) + { + It++; + continue; + } + + std::string_view Remaining{It, size_t(ptrdiff_t(End - It))}; + size_t Idx = Remaining.find(Delim, 0); + + if (Idx == std::string_view::npos) + { + Idx = Remaining.size(); + } + + Count++; + std::string_view Token{It, Idx}; + if (!Func(Token)) + { + break; + } + + It = It + Idx; + } + + return Count; +} + +////////////////////////////////////////////////////////////////////////// + +inline int32_t +StrCaseCompare(const char* Lhs, const char* Rhs, int64_t Length = -1) +{ + // A helper for cross-platform case-insensitive string comparison. +#if ZEN_PLATFORM_WINDOWS + return (Length < 0) ? _stricmp(Lhs, Rhs) : _strnicmp(Lhs, Rhs, size_t(Length)); +#else + return (Length < 0) ? strcasecmp(Lhs, Rhs) : strncasecmp(Lhs, Rhs, size_t(Length)); +#endif +} + +/** + * @brief + * Helper function to implement case sensitive spaceship operator for strings. + * MacOS clang version we use does not implement <=> for std::string + * @param Lhs string + * @param Rhs string + * @return std::strong_ordering indicating relationship between Lhs and Rhs + */ +inline auto +caseSensitiveCompareStrings(const std::string& Lhs, const std::string& Rhs) +{ + int r = Lhs.compare(Rhs); + return r == 0 ? std::strong_ordering::equal : r < 0 ? std::strong_ordering::less : std::strong_ordering::greater; +} + +////////////////////////////////////////////////////////////////////////// + +/** + * ASCII character bitset useful for fast and readable parsing + * + * Entirely constexpr. Works with both wide and narrow strings. + * + * Example use cases: + * + * constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n"); + * bool bIsWhitespace = WhitespaceCharacters.Test(MyChar); + * const char* HelloWorld = AsciiSet::Skip(" \t\tHello world!", WhitespaceCharacters); + * + * constexpr AsciiSet XmlEscapeChars("&<>\"'"); + * check(AsciiSet::HasNone(EscapedXmlString, XmlEscapeChars)); + * + * constexpr AsciiSet Delimiters(".:;"); + * const TCHAR* DelimiterOrEnd = AsciiSet::FindFirstOrEnd(PrefixedName, Delimiters); + * FString Prefix(PrefixedName, DelimiterOrEnd - PrefixedName); + * + * constexpr AsciiSet Slashes("/\\"); + * const TCHAR* SlashOrEnd = AsciiSet::FindLastOrEnd(PathName, Slashes); + * const TCHAR* FileName = *SlashOrEnd ? SlashOrEnd + 1 : PathName; + */ +class AsciiSet +{ +public: + template<typename CharType, int N> + constexpr AsciiSet(const CharType (&Chars)[N]) : AsciiSet(StringToBitset(Chars)) + { + } + + /** Returns true if a character is part of the set */ + template<typename CharType> + constexpr inline bool Contains(CharType Char) const + { + using UnsignedCharType = typename std::make_unsigned<CharType>::type; + + return !!TestImpl((UnsignedCharType)Char); + } + + /** Returns non-zero if a character is part of the set. Prefer Contains() to avoid VS2019 conversion warnings. */ + template<typename CharType> + constexpr inline uint64_t Test(CharType Char) const + { + using UnsignedCharType = typename std::make_unsigned<CharType>::type; + + return TestImpl((UnsignedCharType)Char); + } + + /** Create new set with specified character in it */ + constexpr inline AsciiSet operator+(char Char) const + { + using UnsignedCharType = typename std::make_unsigned<char>::type; + + InitData Bitset = {LoMask, HiMask}; + SetImpl(Bitset, (UnsignedCharType)Char); + return AsciiSet(Bitset); + } + + /** Create new set containing inverse set of characters - likely including null-terminator */ + constexpr inline AsciiSet operator~() const { return AsciiSet(~LoMask, ~HiMask); } + + ////////// Algorithms for C strings ////////// + + /** Find first character of string inside set or end pointer. Never returns null. */ + template<class CharType> + static constexpr const CharType* FindFirstOrEnd(const CharType* Str, AsciiSet Set) + { + for (AsciiSet SetOrNil(Set.LoMask | NilMask, Set.HiMask); !SetOrNil.Test(*Str); ++Str) + ; + + return Str; + } + + /** Find last character of string inside set or end pointer. Never returns null. */ + template<class CharType> + static constexpr const CharType* FindLastOrEnd(const CharType* Str, AsciiSet Set) + { + const CharType* Last = FindFirstOrEnd(Str, Set); + + for (const CharType* It = Last; *It; It = FindFirstOrEnd(It + 1, Set)) + { + Last = It; + } + + return Last; + } + + /** Find first character of string outside of set. Never returns null. */ + template<typename CharType> + static constexpr const CharType* Skip(const CharType* Str, AsciiSet Set) + { + while (Set.Contains(*Str)) + { + ++Str; + } + + return Str; + } + + /** Test if string contains any character in set */ + template<typename CharType> + static constexpr bool HasAny(const CharType* Str, AsciiSet Set) + { + return *FindFirstOrEnd(Str, Set) != '\0'; + } + + /** Test if string contains no character in set */ + template<typename CharType> + static constexpr bool HasNone(const CharType* Str, AsciiSet Set) + { + return *FindFirstOrEnd(Str, Set) == '\0'; + } + + /** Test if string contains any character outside of set */ + template<typename CharType> + static constexpr bool HasOnly(const CharType* Str, AsciiSet Set) + { + return *Skip(Str, Set) == '\0'; + } + + ////////// Algorithms for string types like std::string_view and std::string ////////// + + /** Get initial substring with all characters in set */ + template<class StringType> + static constexpr StringType FindPrefixWith(const StringType& Str, AsciiSet Set) + { + return Scan<EDir::Forward, EInclude::Members, EKeep::Head>(Str, Set); + } + + /** Get initial substring with no characters in set */ + template<class StringType> + static constexpr StringType FindPrefixWithout(const StringType& Str, AsciiSet Set) + { + return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Head>(Str, Set); + } + + /** Trim initial characters in set */ + template<class StringType> + static constexpr StringType TrimPrefixWith(const StringType& Str, AsciiSet Set) + { + return Scan<EDir::Forward, EInclude::Members, EKeep::Tail>(Str, Set); + } + + /** Trim initial characters not in set */ + template<class StringType> + static constexpr StringType TrimPrefixWithout(const StringType& Str, AsciiSet Set) + { + return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Tail>(Str, Set); + } + + /** Get trailing substring with all characters in set */ + template<class StringType> + static constexpr StringType FindSuffixWith(const StringType& Str, AsciiSet Set) + { + return Scan<EDir::Reverse, EInclude::Members, EKeep::Tail>(Str, Set); + } + + /** Get trailing substring with no characters in set */ + template<class StringType> + static constexpr StringType FindSuffixWithout(const StringType& Str, AsciiSet Set) + { + return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Tail>(Str, Set); + } + + /** Trim trailing characters in set */ + template<class StringType> + static constexpr StringType TrimSuffixWith(const StringType& Str, AsciiSet Set) + { + return Scan<EDir::Reverse, EInclude::Members, EKeep::Head>(Str, Set); + } + + /** Trim trailing characters not in set */ + template<class StringType> + static constexpr StringType TrimSuffixWithout(const StringType& Str, AsciiSet Set) + { + return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Head>(Str, Set); + } + + /** Test if string contains any character in set */ + template<class StringType> + static constexpr bool HasAny(const StringType& Str, AsciiSet Set) + { + return !HasNone(Str, Set); + } + + /** Test if string contains no character in set */ + template<class StringType> + static constexpr bool HasNone(const StringType& Str, AsciiSet Set) + { + uint64_t Match = 0; + for (auto Char : Str) + { + Match |= Set.Test(Char); + } + return Match == 0; + } + + /** Test if string contains any character outside of set */ + template<class StringType> + static constexpr bool HasOnly(const StringType& Str, AsciiSet Set) + { + auto End = Str.data() + Str.size(); + return FindFirst<EInclude::Members>(Set, Str.data(), End) == End; + } + +private: + enum class EDir + { + Forward, + Reverse + }; + enum class EInclude + { + Members, + NonMembers + }; + enum class EKeep + { + Head, + Tail + }; + + template<EInclude Include, typename CharType> + static constexpr const CharType* FindFirst(AsciiSet Set, const CharType* It, const CharType* End) + { + for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); ++It) + ; + return It; + } + + template<EInclude Include, typename CharType> + static constexpr const CharType* FindLast(AsciiSet Set, const CharType* It, const CharType* End) + { + for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); --It) + ; + return It; + } + + template<EDir Dir, EInclude Include, EKeep Keep, class StringType> + static constexpr StringType Scan(const StringType& Str, AsciiSet Set) + { + auto Begin = Str.data(); + auto End = Begin + Str.size(); + auto It = Dir == EDir::Forward ? FindFirst<Include>(Set, Begin, End) : FindLast<Include>(Set, End - 1, Begin - 1) + 1; + + return Keep == EKeep::Head ? StringType(Begin, static_cast<int32_t>(It - Begin)) : StringType(It, static_cast<int32_t>(End - It)); + } + + // Work-around for constexpr limitations + struct InitData + { + uint64_t Lo, Hi; + }; + static constexpr uint64_t NilMask = uint64_t(1) << '\0'; + + static constexpr inline void SetImpl(InitData& Bitset, uint32_t Char) + { + uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0); + uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1); + uint64_t Bit = uint64_t(1) << uint8_t(Char & 0x3f); + + Bitset.Lo |= Bit & IsLo; + Bitset.Hi |= Bit & IsHi; + } + + constexpr inline uint64_t TestImpl(uint32_t Char) const + { + uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0); + uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1); + uint64_t Bit = uint64_t(1) << (Char & 0x3f); + + return (Bit & IsLo & LoMask) | (Bit & IsHi & HiMask); + } + + template<typename CharType, int N> + static constexpr InitData StringToBitset(const CharType (&Chars)[N]) + { + using UnsignedCharType = typename std::make_unsigned<CharType>::type; + + InitData Bitset = {0, 0}; + for (int I = 0; I < N - 1; ++I) + { + SetImpl(Bitset, UnsignedCharType(Chars[I])); + } + + return Bitset; + } + + constexpr AsciiSet(InitData Bitset) : LoMask(Bitset.Lo), HiMask(Bitset.Hi) {} + + constexpr AsciiSet(uint64_t Lo, uint64_t Hi) : LoMask(Lo), HiMask(Hi) {} + + uint64_t LoMask, HiMask; +}; + +////////////////////////////////////////////////////////////////////////// + +void string_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/testing.h b/src/zencore/include/zencore/testing.h new file mode 100644 index 000000000..a00ee3166 --- /dev/null +++ b/src/zencore/include/zencore/testing.h @@ -0,0 +1,67 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <memory> + +#ifndef ZEN_TEST_WITH_RUNNER +# define ZEN_TEST_WITH_RUNNER 0 +#endif + +#if ZEN_TEST_WITH_RUNNER +# define DOCTEST_CONFIG_IMPLEMENT +#endif + +#if ZEN_WITH_TESTS +# include <doctest/doctest.h> +inline auto +Approx(auto Value) +{ + return doctest::Approx(Value); +} +#endif + +/** + * Test runner helper + * + * This acts as a thin layer between the test app and the test + * framework, which is used to customize configuration logic + * and to set up logging. + * + * If you don't want to implement custom setup then the + * ZEN_RUN_TESTS macro can be used instead. + */ + +#if ZEN_WITH_TESTS +namespace zen::testing { + +class TestRunner +{ +public: + TestRunner(); + ~TestRunner(); + + int ApplyCommandLine(int argc, char const* const* argv); + int Run(); + +private: + struct Impl; + + std::unique_ptr<Impl> m_Impl; +}; + +# define ZEN_RUN_TESTS(argC, argV) \ + [&] { \ + zen::testing::TestRunner Runner; \ + Runner.ApplyCommandLine(argC, argV); \ + return Runner.Run(); \ + }() + +} // namespace zen::testing +#endif + +#if ZEN_TEST_WITH_RUNNER +# undef DOCTEST_CONFIG_IMPLEMENT +#endif diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h new file mode 100644 index 000000000..04648c6de --- /dev/null +++ b/src/zencore/include/zencore/testutils.h @@ -0,0 +1,32 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <filesystem> + +namespace zen { + +std::filesystem::path CreateTemporaryDirectory(); + +class ScopedTemporaryDirectory +{ +public: + explicit ScopedTemporaryDirectory(std::filesystem::path Directory); + ScopedTemporaryDirectory(); + ~ScopedTemporaryDirectory(); + + std::filesystem::path& Path() { return m_RootPath; } + +private: + std::filesystem::path m_RootPath; +}; + +struct ScopedCurrentDirectoryChange +{ + std::filesystem::path OldPath{std::filesystem::current_path()}; + + ScopedCurrentDirectoryChange() { std::filesystem::current_path(CreateTemporaryDirectory()); } + ~ScopedCurrentDirectoryChange() { std::filesystem::current_path(OldPath); } +}; + +} // namespace zen diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h new file mode 100644 index 000000000..a9c96d422 --- /dev/null +++ b/src/zencore/include/zencore/thread.h @@ -0,0 +1,273 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <atomic> +#include <filesystem> +#include <shared_mutex> +#include <string_view> +#include <vector> + +namespace zen { + +void SetCurrentThreadName(std::string_view ThreadName); + +/** + * Reader-writer lock + * + * - A single thread may hold an exclusive lock at any given moment + * + * - Multiple threads may hold shared locks, but only if no thread has + * acquired an exclusive lock + */ +class RwLock +{ +public: + ZENCORE_API void AcquireShared(); + ZENCORE_API void ReleaseShared(); + + ZENCORE_API void AcquireExclusive(); + ZENCORE_API void ReleaseExclusive(); + + struct SharedLockScope + { + SharedLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireShared(); } + ~SharedLockScope() { ReleaseNow(); } + + void ReleaseNow() + { + if (m_Lock) + { + m_Lock->ReleaseShared(); + m_Lock = nullptr; + } + } + + private: + RwLock* m_Lock; + }; + + struct ExclusiveLockScope + { + ExclusiveLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireExclusive(); } + ~ExclusiveLockScope() { ReleaseNow(); } + + void ReleaseNow() + { + if (m_Lock) + { + m_Lock->ReleaseExclusive(); + m_Lock = nullptr; + } + } + + private: + RwLock* m_Lock; + }; + +private: + std::shared_mutex m_Mutex; +}; + +/** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore') + */ +class Event +{ +public: + ZENCORE_API Event(); + ZENCORE_API ~Event(); + + Event(Event&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; } + + Event(const Event& Rhs) = delete; + Event& operator=(const Event& Rhs) = delete; + + inline Event& operator=(Event&& Rhs) noexcept + { + std::swap(m_EventHandle, Rhs.m_EventHandle); + return *this; + } + + ZENCORE_API void Set(); + ZENCORE_API void Reset(); + ZENCORE_API bool Wait(int TimeoutMs = -1); + ZENCORE_API void Close(); + +protected: + explicit Event(void* EventHandle) : m_EventHandle(EventHandle) {} + + void* m_EventHandle = nullptr; +}; + +/** Basic abstraction of an IPC mechanism (aka 'binary semaphore') + */ +class NamedEvent +{ +public: + NamedEvent() = default; + ZENCORE_API explicit NamedEvent(std::string_view EventName); + ZENCORE_API ~NamedEvent(); + ZENCORE_API void Close(); + ZENCORE_API void Set(); + ZENCORE_API bool Wait(int TimeoutMs = -1); + + NamedEvent(NamedEvent&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; } + + inline NamedEvent& operator=(NamedEvent&& Rhs) noexcept + { + std::swap(m_EventHandle, Rhs.m_EventHandle); + return *this; + } + +protected: + void* m_EventHandle = nullptr; + +private: + NamedEvent(const NamedEvent& Rhs) = delete; + NamedEvent& operator=(const NamedEvent& Rhs) = delete; +}; + +/** Basic abstraction of a named (system wide) mutex primitive + */ +class NamedMutex +{ +public: + ~NamedMutex(); + + ZENCORE_API [[nodiscard]] bool Create(std::string_view MutexName); + + ZENCORE_API static bool Exists(std::string_view MutexName); + +private: + void* m_MutexHandle = nullptr; +}; + +/** + * Downward counter of type std::ptrdiff_t which can be used to synchronize threads + */ +class Latch +{ +public: + Latch(std::ptrdiff_t Count) : Counter(Count) {} + + void CountDown() + { + std::ptrdiff_t Old = Counter.fetch_sub(1); + if (Old == 1) + { + Complete.Set(); + } + } + + std::ptrdiff_t Remaining() const { return Counter.load(); } + + // If you want to add dynamic count, make sure to set the initial counter to 1 + // and then do a CountDown() just before wait to not trigger the event causing + // false positive completion results. + void AddCount(std::ptrdiff_t Count) + { + std::atomic_ptrdiff_t Old = Counter.fetch_add(Count); + ZEN_ASSERT_SLOW(Old > 0); + } + + bool Wait(int TimeoutMs = -1) + { + std::ptrdiff_t Old = Counter.load(); + if (Old == 0) + { + return true; + } + return Complete.Wait(TimeoutMs); + } + +private: + std::atomic_ptrdiff_t Counter; + Event Complete; +}; + +/** Basic process abstraction + */ +class ProcessHandle +{ +public: + ZENCORE_API ProcessHandle(); + + ProcessHandle(const ProcessHandle&) = delete; + ProcessHandle& operator=(const ProcessHandle&) = delete; + + ZENCORE_API ~ProcessHandle(); + + ZENCORE_API void Initialize(int Pid); + ZENCORE_API void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle + ZENCORE_API [[nodiscard]] bool IsRunning() const; + ZENCORE_API [[nodiscard]] bool IsValid() const; + ZENCORE_API bool Wait(int TimeoutMs = -1); + ZENCORE_API void Terminate(int ExitCode); + ZENCORE_API void Reset(); + [[nodiscard]] inline int Pid() const { return m_Pid; } + +private: + void* m_ProcessHandle = nullptr; + int m_Pid = 0; +}; + +/** Basic process creation + */ +struct CreateProcOptions +{ + enum + { + Flag_NewConsole = 1 << 0, + Flag_Elevated = 1 << 1, + Flag_Unelevated = 1 << 2, + }; + + const std::filesystem::path* WorkingDirectory = nullptr; + uint32_t Flags = 0; +}; + +#if ZEN_PLATFORM_WINDOWS +using CreateProcResult = void*; // handle to the process +#else +using CreateProcResult = int32_t; // pid +#endif + +ZENCORE_API CreateProcResult CreateProc(const std::filesystem::path& Executable, + std::string_view CommandLine, // should also include arg[0] (executable name) + const CreateProcOptions& Options = {}); + +/** Process monitor - monitors a list of running processes via polling + + Intended to be used to monitor a set of "sponsor" processes, where + we need to determine when none of them remain alive + + */ + +class ProcessMonitor +{ +public: + ProcessMonitor(); + ~ProcessMonitor(); + + ZENCORE_API bool IsRunning(); + ZENCORE_API void AddPid(int Pid); + ZENCORE_API bool IsActive() const; + +private: + using HandleType = void*; + + mutable RwLock m_Lock; + std::vector<HandleType> m_ProcessHandles; +}; + +ZENCORE_API bool IsProcessRunning(int pid); +ZENCORE_API int GetCurrentProcessId(); +ZENCORE_API int GetCurrentThreadId(); + +ZENCORE_API void Sleep(int ms); + +void thread_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/timer.h b/src/zencore/include/zencore/timer.h new file mode 100644 index 000000000..e4ddc3505 --- /dev/null +++ b/src/zencore/include/zencore/timer.h @@ -0,0 +1,58 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#if ZEN_COMPILER_MSC +# include <intrin.h> +#elif ZEN_ARCH_X64 +# include <x86intrin.h> +#endif + +#include <stdint.h> + +namespace zen { + +// High frequency timers + +ZENCORE_API uint64_t GetHifreqTimerValue(); +ZENCORE_API uint64_t GetHifreqTimerFrequency(); +ZENCORE_API double GetHifreqTimerToSeconds(); +ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init + +class Stopwatch +{ +public: + inline Stopwatch() : m_StartValue(GetHifreqTimerValue()) {} + + inline uint64_t GetElapsedTimeMs() const { return (GetHifreqTimerValue() - m_StartValue) * 1'000 / GetHifreqTimerFrequency(); } + inline uint64_t GetElapsedTimeUs() const { return (GetHifreqTimerValue() - m_StartValue) * 1'000'000 / GetHifreqTimerFrequency(); } + inline uint64_t GetElapsedTicks() const { return GetHifreqTimerValue() - m_StartValue; } + inline void Reset() { m_StartValue = GetHifreqTimerValue(); } + + static inline uint64_t GetElapsedTimeMs(uint64_t Ticks) { return Ticks * 1'000 / GetHifreqTimerFrequency(); } + static inline uint64_t GetElapsedTimeUs(uint64_t Ticks) { return Ticks * 1'000'000 / GetHifreqTimerFrequency(); } + +private: + uint64_t m_StartValue; +}; + +// Low frequency timers + +namespace detail { + extern ZENCORE_API uint64_t g_LofreqTimerValue; +} // namespace detail + +inline uint64_t +GetLofreqTimerValue() +{ + return detail::g_LofreqTimerValue; +} + +ZENCORE_API void UpdateLofreqTimerValue(); +ZENCORE_API uint64_t GetLofreqTimerFrequency(); + +void timer_forcelink(); // internal + +} // namespace zen diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h new file mode 100644 index 000000000..0af490f23 --- /dev/null +++ b/src/zencore/include/zencore/trace.h @@ -0,0 +1,36 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +/* clang-format off */ + +#include <zencore/zencore.h> + +#if ZEN_WITH_TRACE + +ZEN_THIRD_PARTY_INCLUDES_START +#if !defined(TRACE_IMPLEMENT) +# define TRACE_IMPLEMENT 0 +#endif +#include <trace.h> +#undef TRACE_IMPLEMENT +ZEN_THIRD_PARTY_INCLUDES_END + +#define ZEN_TRACE_CPU(x) TRACE_CPU_SCOPE(x) + +enum class TraceType +{ + File, + Network, + None +}; + +void TraceInit(const char* HostOrPath, TraceType Type); + +#else + +#define ZEN_TRACE_CPU(x) + +#endif // ZEN_WITH_TRACE + +/* clang-format on */ diff --git a/src/zencore/include/zencore/uid.h b/src/zencore/include/zencore/uid.h new file mode 100644 index 000000000..9659f5893 --- /dev/null +++ b/src/zencore/include/zencore/uid.h @@ -0,0 +1,87 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> +#include <compare> + +namespace zen { + +class StringBuilderBase; + +/** Object identifier + + Can be used as a GUID essentially, but is more compact (12 bytes) and as such + is more susceptible to collisions than a 16-byte GUID but also I don't expect + the population to be large so in practice the risk should be minimal due to + how the identifiers work. + + Similar in spirit to MongoDB ObjectId + + When serialized, object identifiers generated in a given session in sequence + will sort in chronological order since the timestamp is in the MSB in big + endian format. This makes it suitable as a database key since most indexing + structures work better when keys are inserted in lexicographically + increasing order. + + The current layout is basically: + + |----------------|----------------|----------------| + | timestamp | serial # | run id | + |----------------|----------------|----------------| + MSB LSB + + - Timestamp is a unsigned 32-bit value (seconds since 00:00:00 Jan 1 2021) + - Serial # is another unsigned 32-bit value which is assigned a (strong) + random number at initialization time which is incremented when a new Oid + is generated + - The run id is generated from a strong random number generator + at initialization time and stays fixed for the duration of the program + + Timestamp and serial are stored in memory in such a way that they can be + ordered lexicographically. I.e they are in big-endian byte order. + + NOTE: The information above is only meant to explain the properties of + the identifiers. Client code should simply treat the identifier as an + opaque value and may not make any assumptions on the structure, as there + may be other ways of generating the identifiers in the future if an + application benefits. + + */ + +struct Oid +{ + static const int StringLength = 24; + typedef char String_t[StringLength + 1]; + + static void Initialize(); + [[nodiscard]] static Oid NewOid(); + + const Oid& Generate(); + [[nodiscard]] static Oid FromHexString(const std::string_view String); + StringBuilderBase& ToString(StringBuilderBase& OutString) const; + void ToString(char OutString[StringLength]); + [[nodiscard]] static Oid FromMemory(const void* Ptr); + + auto operator<=>(const Oid& rhs) const = default; + [[nodiscard]] inline operator bool() const { return *this != Zero; } + + static const Oid Zero; // Min (can be used to signify a "null" value, or for open range queries) + static const Oid Max; // Max (can be used for open range queries) + + struct Hasher + { + size_t operator()(const Oid& id) const + { + const size_t seed = id.OidBits[0]; + return ((seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1])) | (uint64_t(id.OidBits[2]) << 32); + } + }; + + // You should not assume anything about these words + uint32_t OidBits[3]; +}; + +extern void uid_forcelink(); + +} // namespace zen diff --git a/src/zencore/include/zencore/varint.h b/src/zencore/include/zencore/varint.h new file mode 100644 index 000000000..e57e1d497 --- /dev/null +++ b/src/zencore/include/zencore/varint.h @@ -0,0 +1,277 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "intmath.h" + +#include <algorithm> + +namespace zen { + +// Variable-Length Integer Encoding +// +// ZigZag encoding is used to convert signed integers into unsigned integers in a way that allows +// integers with a small magnitude to have a smaller encoded representation. +// +// An unsigned integer is encoded into 1-9 bytes based on its magnitude. The first byte indicates +// how many additional bytes are used by the number of leading 1-bits that it has. The additional +// bytes are stored in big endian order, and the most significant bits of the value are stored in +// the remaining bits in the first byte. The encoding of the first byte allows the reader to skip +// over the encoded integer without consuming its bytes individually. +// +// Encoded unsigned integers sort the same in a byte-wise comparison as when their decoded values +// are compared. The same property does not hold for signed integers due to ZigZag encoding. +// +// 32-bit inputs encode to 1-5 bytes. +// 64-bit inputs encode to 1-9 bytes. +// +// 0x0000'0000'0000'0000 - 0x0000'0000'0000'007f : 0b0_______ 1 byte +// 0x0000'0000'0000'0080 - 0x0000'0000'0000'3fff : 0b10______ 2 bytes +// 0x0000'0000'0000'4000 - 0x0000'0000'001f'ffff : 0b110_____ 3 bytes +// 0x0000'0000'0020'0000 - 0x0000'0000'0fff'ffff : 0b1110____ 4 bytes +// 0x0000'0000'1000'0000 - 0x0000'0007'ffff'ffff : 0b11110___ 5 bytes +// 0x0000'0008'0000'0000 - 0x0000'03ff'ffff'ffff : 0b111110__ 6 bytes +// 0x0000'0400'0000'0000 - 0x0001'ffff'ffff'ffff : 0b1111110_ 7 bytes +// 0x0002'0000'0000'0000 - 0x00ff'ffff'ffff'ffff : 0b11111110 8 bytes +// 0x0100'0000'0000'0000 - 0xffff'ffff'ffff'ffff : 0b11111111 9 bytes +// +// Encoding Examples +// -42 => ZigZag => 0x53 => 0x53 +// 42 => ZigZag => 0x54 => 0x54 +// 0x1 => 0x01 +// 0x12 => 0x12 +// 0x123 => 0x81 0x23 +// 0x1234 => 0x92 0x34 +// 0x12345 => 0xc1 0x23 0x45 +// 0x123456 => 0xd2 0x34 0x56 +// 0x1234567 => 0xe1 0x23 0x45 0x67 +// 0x12345678 => 0xf0 0x12 0x34 0x56 0x78 +// 0x123456789 => 0xf1 0x23 0x45 0x67 0x89 +// 0x123456789a => 0xf8 0x12 0x34 0x56 0x78 0x9a +// 0x123456789ab => 0xfb 0x23 0x45 0x67 0x89 0xab +// 0x123456789abc => 0xfc 0x12 0x34 0x56 0x78 0x9a 0xbc +// 0x123456789abcd => 0xfd 0x23 0x45 0x67 0x89 0xab 0xcd +// 0x123456789abcde => 0xfe 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde +// 0x123456789abcdef => 0xff 0x01 0x23 0x45 0x67 0x89 0xab 0xcd 0xef +// 0x123456789abcdef0 => 0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0 + +/** + * Measure the length in bytes (1-9) of an encoded variable-length integer. + * + * @param InData A variable-length encoding of an (signed or unsigned) integer. + * @return The number of bytes used to encode the integer, in the range 1-9. + */ +inline uint32_t +MeasureVarUInt(const void* InData) +{ + return CountLeadingZeros(uint8_t(~*static_cast<const uint8_t*>(InData))) - 23; +} + +/** Measure the length in bytes (1-9) of an encoded variable-length integer. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(const void* InData) +{ + return MeasureVarUInt(InData); +} + +/** Measure the number of bytes (1-5) required to encode the 32-bit input. */ +inline uint32_t +MeasureVarUInt(uint32_t InValue) +{ + return uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1); +} + +/** Measure the number of bytes (1-9) required to encode the 64-bit input. */ +inline uint32_t +MeasureVarUInt(uint64_t InValue) +{ + return uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9)); +} + +/** Measure the number of bytes (1-5) required to encode the 32-bit input. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(int32_t InValue) +{ + return MeasureVarUInt(uint32_t((InValue >> 31) ^ (InValue << 1))); +} + +/** Measure the number of bytes (1-9) required to encode the 64-bit input. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(int64_t InValue) +{ + return MeasureVarUInt(uint64_t((InValue >> 63) ^ (InValue << 1))); +} + +/** + * Read a variable-length unsigned integer. + * + * @param InData A variable-length encoding of an unsigned integer. + * @param OutByteCount The number of bytes consumed from the input. + * @return An unsigned integer. + */ +inline uint64_t +ReadVarUInt(const void* InData, uint32_t& OutByteCount) +{ + const uint32_t ByteCount = MeasureVarUInt(InData); + OutByteCount = ByteCount; + + const uint8_t* InBytes = static_cast<const uint8_t*>(InData); + uint64_t Value = *InBytes++ & uint8_t(0xff >> ByteCount); + switch (ByteCount - 1) + { + case 8: + Value <<= 8; + Value |= *InBytes++; + [[fallthrough]]; + case 7: + Value <<= 8; + Value |= *InBytes++; + [[fallthrough]]; + case 6: + Value <<= 8; + Value |= *InBytes++; + [[fallthrough]]; + case 5: + Value <<= 8; + Value |= *InBytes++; + [[fallthrough]]; + case 4: + Value <<= 8; + Value |= *InBytes++; + [[fallthrough]]; + case 3: + Value <<= 8; + Value |= *InBytes++; + [[fallthrough]]; + case 2: + Value <<= 8; + Value |= *InBytes++; + [[fallthrough]]; + case 1: + Value <<= 8; + Value |= *InBytes++; + [[fallthrough]]; + default: + return Value; + } +} + +/** + * Read a variable-length signed integer. + * + * @param InData A variable-length encoding of a signed integer. + * @param OutByteCount The number of bytes consumed from the input. + * @return A signed integer. + */ +inline int64_t +ReadVarInt(const void* InData, uint32_t& OutByteCount) +{ + const uint64_t Value = ReadVarUInt(InData, OutByteCount); + return -int64_t(Value & 1) ^ int64_t(Value >> 1); +} + +/** + * Write a variable-length unsigned integer. + * + * @param InValue An unsigned integer to encode. + * @param OutData A buffer of at least 5 bytes to write the output to. + * @return The number of bytes used in the output. + */ +inline uint32_t +WriteVarUInt(uint32_t InValue, void* OutData) +{ + const uint32_t ByteCount = MeasureVarUInt(InValue); + uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1; + switch (ByteCount - 1) + { + case 4: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 3: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 2: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 1: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + default: + break; + } + *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue); + return ByteCount; +} + +/** + * Write a variable-length unsigned integer. + * + * @param InValue An unsigned integer to encode. + * @param OutData A buffer of at least 9 bytes to write the output to. + * @return The number of bytes used in the output. + */ +inline uint32_t +WriteVarUInt(uint64_t InValue, void* OutData) +{ + const uint32_t ByteCount = MeasureVarUInt(InValue); + uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1; + switch (ByteCount - 1) + { + case 8: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 7: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 6: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 5: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 4: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 3: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 2: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + case 1: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + [[fallthrough]]; + default: + break; + } + *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue); + return ByteCount; +} + +/** Write a variable-length signed integer. \see \ref WriteVarUInt */ +inline uint32_t +WriteVarInt(int32_t InValue, void* OutData) +{ + const uint32_t Value = uint32_t((InValue >> 31) ^ (InValue << 1)); + return WriteVarUInt(Value, OutData); +} + +/** Write a variable-length signed integer. \see \ref WriteVarUInt */ +inline uint32_t +WriteVarInt(int64_t InValue, void* OutData) +{ + const uint64_t Value = uint64_t((InValue >> 63) ^ (InValue << 1)); + return WriteVarUInt(Value, OutData); +} + +} // namespace zen diff --git a/src/zencore/include/zencore/windows.h b/src/zencore/include/zencore/windows.h new file mode 100644 index 000000000..91828f0ec --- /dev/null +++ b/src/zencore/include/zencore/windows.h @@ -0,0 +1,25 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +ZEN_THIRD_PARTY_INCLUDES_START + +struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- +#ifndef NOMINMAX +# define NOMINMAX // We don't want your min/max macros +#endif +#ifndef NOGDI +# define NOGDI // We don't want your GetObject define +#endif +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif +#ifndef _WIN32_WINNT +# define _WIN32_WINNT 0x0A00 +#endif +#include <windows.h> +#undef GetObject + +ZEN_THIRD_PARTY_INCLUDES_END diff --git a/src/zencore/include/zencore/workthreadpool.h b/src/zencore/include/zencore/workthreadpool.h new file mode 100644 index 000000000..0ddc65298 --- /dev/null +++ b/src/zencore/include/zencore/workthreadpool.h @@ -0,0 +1,48 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/blockingqueue.h> +#include <zencore/refcount.h> + +#include <exception> +#include <functional> +#include <system_error> +#include <thread> +#include <vector> + +namespace zen { + +struct IWork : public RefCounted +{ + virtual void Execute() = 0; + + inline std::exception_ptr GetException() { return m_Exception; } + +private: + std::exception_ptr m_Exception; + + friend class WorkerThreadPool; +}; + +class WorkerThreadPool +{ +public: + WorkerThreadPool(int InThreadCount); + ~WorkerThreadPool(); + + void ScheduleWork(Ref<IWork> Work); + void ScheduleWork(std::function<void()>&& Work); + + [[nodiscard]] size_t PendingWork() const; + +private: + void WorkerThreadFunction(); + + std::vector<std::thread> m_WorkerThreads; + BlockingQueue<Ref<IWork>> m_WorkQueue; +}; + +} // namespace zen diff --git a/src/zencore/include/zencore/xxhash.h b/src/zencore/include/zencore/xxhash.h new file mode 100644 index 000000000..04872f4c3 --- /dev/null +++ b/src/zencore/include/zencore/xxhash.h @@ -0,0 +1,89 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/memory.h> + +#include <xxh3.h> + +#include <compare> +#include <string_view> + +namespace zen { + +class StringBuilderBase; + +/** + * XXH3 hash + */ +struct XXH3_128 +{ + uint8_t Hash[16]; + + static XXH3_128 MakeFrom(const void* data /* 16 bytes */) + { + XXH3_128 Xx; + memcpy(Xx.Hash, data, sizeof Xx); + return Xx; + } + + static inline XXH3_128 HashMemory(const void* data, size_t byteCount) + { + XXH3_128 Hash; + XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits(data, byteCount)); + return Hash; + } + static XXH3_128 HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); } + static XXH3_128 FromHexString(const char* string); + static XXH3_128 FromHexString(const std::string_view string); + const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static const int StringLength = 32; + typedef char String_t[StringLength + 1]; + + static XXH3_128 Zero; // Initialized to all zeros + + inline auto operator<=>(const XXH3_128& rhs) const = default; + + struct Hasher + { + size_t operator()(const XXH3_128& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct XXH3_128Stream +{ + /// Begin streaming hash compute (not needed on freshly constructed instance) + void Reset() { memset(&m_State, 0, sizeof m_State); } + + /// Append another chunk + XXH3_128Stream& Append(const void* Data, size_t ByteCount) + { + XXH3_128bits_update(&m_State, Data, ByteCount); + return *this; + } + + /// Append another chunk + XXH3_128Stream& Append(MemoryView Data) { return Append(Data.GetData(), Data.GetSize()); } + + /// Obtain final hash. If you wish to reuse the instance call reset() + XXH3_128 GetHash() + { + XXH3_128 Hash; + XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits_digest(&m_State)); + return Hash; + } + +private: + XXH3_state_s m_State{}; +}; + +} // namespace zen diff --git a/src/zencore/include/zencore/zencore.h b/src/zencore/include/zencore/zencore.h new file mode 100644 index 000000000..5bcd77239 --- /dev/null +++ b/src/zencore/include/zencore/zencore.h @@ -0,0 +1,383 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <cinttypes> +#include <stdexcept> +#include <string> +#include <version> + +#ifndef ZEN_WITH_TESTS +# define ZEN_WITH_TESTS 1 +#endif + +////////////////////////////////////////////////////////////////////////// +// Platform +// + +#define ZEN_PLATFORM_WINDOWS 0 +#define ZEN_PLATFORM_LINUX 0 +#define ZEN_PLATFORM_MAC 0 + +#ifdef _WIN32 +# undef ZEN_PLATFORM_WINDOWS +# define ZEN_PLATFORM_WINDOWS 1 +#elif defined(__linux__) +# undef ZEN_PLATFORM_LINUX +# define ZEN_PLATFORM_LINUX 1 +#elif defined(__APPLE__) +# undef ZEN_PLATFORM_MAC +# define ZEN_PLATFORM_MAC 1 +#endif + +#if ZEN_PLATFORM_WINDOWS +# if !defined(NOMINMAX) +# define NOMINMAX // stops Windows.h from defining 'min/max' macros +# endif +# if !defined(NOGDI) +# define NOGDI +# endif +# if !defined(WIN32_LEAN_AND_MEAN) +# define WIN32_LEAN_AND_MEAN // cut-down what Windows.h defines +# endif +#endif + +////////////////////////////////////////////////////////////////////////// +// Compiler +// + +#define ZEN_COMPILER_CLANG 0 +#define ZEN_COMPILER_MSC 0 +#define ZEN_COMPILER_GCC 0 + +// Clang can define __GNUC__ and/or _MSC_VER so we check for Clang first +#ifdef __clang__ +# undef ZEN_COMPILER_CLANG +# define ZEN_COMPILER_CLANG 1 +#elif defined(_MSC_VER) +# undef ZEN_COMPILER_MSC +# define ZEN_COMPILER_MSC 1 +#elif defined(__GNUC__) +# undef ZEN_COMPILER_GCC +# define ZEN_COMPILER_GCC 1 +#else +# error Unknown compiler +#endif + +#if ZEN_COMPILER_MSC +# pragma warning(disable : 4324) // warning C4324: '<type>': structure was padded due to alignment specifier +# pragma warning(default : 4668) // warning C4668: 'symbol' is not defined as a preprocessor macro, replacing with '0' for 'directives' +# pragma warning(default : 4100) // warning C4100: 'identifier' : unreferenced formal parameter +#endif + +#ifndef ZEN_THIRD_PARTY_INCLUDES_START +# if ZEN_COMPILER_MSC +# define ZEN_THIRD_PARTY_INCLUDES_START \ + __pragma(warning(push)) __pragma(warning(disable : 4668)) /* C4668: use of undefined preprocessor macro */ \ + __pragma(warning(disable : 4305)) /* C4305: 'if': truncation from 'uint32' to 'bool' */ \ + __pragma(warning(disable : 4267)) /* C4267: '=': conversion from 'size_t' to 'US' */ \ + __pragma(warning(disable : 4127)) /* C4127: conditional expression is constant */ \ + __pragma(warning(disable : 4189)) /* C4189: local variable is initialized but not referenced */ +# elif ZEN_COMPILER_CLANG +# define ZEN_THIRD_PARTY_INCLUDES_START \ + _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wundef\"") \ + _Pragma("clang diagnostic ignored \"-Wunused-parameter\"") _Pragma("clang diagnostic ignored \"-Wunused-variable\"") +# elif ZEN_COMPILER_GCC +# define ZEN_THIRD_PARTY_INCLUDES_START \ + _Pragma("GCC diagnostic push") /* NB. ignoring -Wundef doesn't work with GCC */ \ + _Pragma("GCC diagnostic ignored \"-Wunused-parameter\"") _Pragma("GCC diagnostic ignored \"-Wunused-variable\"") +# endif +#endif + +#ifndef ZEN_THIRD_PARTY_INCLUDES_END +# if ZEN_COMPILER_MSC +# define ZEN_THIRD_PARTY_INCLUDES_END __pragma(warning(pop)) +# elif ZEN_COMPILER_CLANG +# define ZEN_THIRD_PARTY_INCLUDES_END _Pragma("clang diagnostic pop") +# elif ZEN_COMPILER_GCC +# define ZEN_THIRD_PARTY_INCLUDES_END _Pragma("GCC diagnostic pop") +# endif +#endif + +#if ZEN_COMPILER_MSC +# define ZEN_DEBUG_BREAK() \ + do \ + { \ + __debugbreak(); \ + } while (0) +#else +# define ZEN_DEBUG_BREAK() \ + do \ + { \ + __builtin_trap(); \ + } while (0) +#endif + +////////////////////////////////////////////////////////////////////////// +// C++20 support +// + +// Clang +#if ZEN_COMPILER_CLANG && __clang_major__ < 12 +# error clang-12 onwards is required for C++20 support +#endif + +// GCC +#if ZEN_COMPILER_GCC && __GNUC__ < 11 +# error GCC-11 onwards is required for C++20 support +#endif + +// GNU libstdc++ +#if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11 +# error GNU libstdc++-11 onwards is required for C++20 support +#endif + +// LLVM libc++ +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 12000 +# error LLVM libc++-12 onwards is required for C++20 support +#endif + +// At the time of writing only ver >= 13 of LLVM's libc++ has an implementation +// of std::integral. Some platforms like Ubuntu and Mac OS are still on 12. +#if defined(__cpp_lib_concepts) +# include <concepts> +template<class T> +concept Integral = std::integral<T>; +template<class T> +concept SignedIntegral = std::signed_integral<T>; +template<class T> +concept UnsignedIntegral = std::unsigned_integral<T>; +template<class F, class... A> +concept Invocable = std::invocable<F, A...>; +template<class D, class B> +concept DerivedFrom = std::derived_from<D, B>; +#else +template<class T> +concept Integral = std::is_integral_v<T>; +template<class T> +concept SignedIntegral = Integral<T> && std::is_signed_v<T>; +template<class T> +concept UnsignedIntegral = Integral<T> && !std::is_signed_v<T>; +template<class F, class... A> +concept Invocable = requires(F&& f, A&&... a) +{ + std::invoke(std::forward<F>(f), std::forward<A>(a)...); +}; +template<class D, class B> +concept DerivedFrom = std::is_base_of_v<B, D> && std::is_convertible_v<const volatile D*, const volatile B*>; +#endif + +#if defined(__cpp_lib_ranges) +template<typename T> +concept ContiguousRange = std::ranges::contiguous_range<T>; +#else +template<typename T> +concept ContiguousRange = true; +#endif + +////////////////////////////////////////////////////////////////////////// +// Architecture +// + +#if defined(__amd64__) || defined(_M_X64) +# define ZEN_ARCH_X64 1 +# define ZEN_ARCH_ARM64 0 +#elif defined(__arm64__) || defined(_M_ARM64) +# define ZEN_ARCH_X64 0 +# define ZEN_ARCH_ARM64 1 +#else +# error Unknown architecture +#endif + +////////////////////////////////////////////////////////////////////////// +// Build flavor +// + +#ifdef NDEBUG +# define ZEN_BUILD_DEBUG 0 +# define ZEN_BUILD_RELEASE 1 +#else +# define ZEN_BUILD_DEBUG 1 +# define ZEN_BUILD_RELEASE 0 +#endif + +////////////////////////////////////////////////////////////////////////// + +#define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1 + +#if defined(__SIZEOF_WCHAR_T__) && __SIZEOF_WCHAR_T__ == 4 +# define ZEN_SIZEOF_WCHAR_T 4 +#else +static_assert(sizeof(wchar_t) == 2, "wchar_t is expected to be two bytes in size"); +# define ZEN_SIZEOF_WCHAR_T 2 +#endif + +////////////////////////////////////////////////////////////////////////// +// Assert +// + +#if ZEN_PLATFORM_WINDOWS +// Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable. +# define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name)) +# define ZEN_FORCENOINLINE __declspec(noinline) /* Force code to NOT be inline */ +# define LINE_TERMINATOR_ANSI "\r\n" +#else +# define ZEN_CODE_SECTION(Name) +# define ZEN_FORCENOINLINE +# define LINE_TERMINATOR_ANSI "\n" +#endif + +#if ZEN_ARCH_ARM64 +// On ARM we can't do this because the executable will require jumps larger +// than the branch instruction can handle. Clang will only generate +// the trampolines in the .text segment of the binary. If the zcold segment +// is present it will generate code that it cannot link. +# define ZEN_DEBUG_SECTION +#else +// We'll put all assert implementation code into a separate section in the linked +// executable. This code should never execute so using a separate section keeps +// it well off the hot path and hopefully out of the instruction cache. It also +// facilitates reasoning about the makeup of a compiled/linked binary. +# define ZEN_DEBUG_SECTION ZEN_CODE_SECTION(".zcold") +#endif + +namespace zen +{ + class AssertException : public std::logic_error + { + public: + AssertException(const char* Msg) : std::logic_error(Msg) {} + }; + + struct AssertImpl + { + static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecAssert + [[noreturn]] (const char* Filename, int LineNumber, const char* FunctionName, const char* Msg) + { + CurrentAssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg); + throw AssertException{Msg}; + } + + protected: + virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION OnAssert(const char* Filename, + int LineNumber, + const char* FunctionName, + const char* Msg) + { + (void(Filename)); + (void(LineNumber)); + (void(FunctionName)); + (void(Msg)); + } + static AssertImpl DefaultAssertImpl; + static AssertImpl* CurrentAssertImpl; + }; + +} // namespace zen + +#define ZEN_ASSERT(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ + } while (false) + +#ifndef NDEBUG +# define ZEN_ASSERT_SLOW(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ + } while (false) +#else +# define ZEN_ASSERT_SLOW(x, ...) +#endif + +////////////////////////////////////////////////////////////////////////// + +#ifdef __clang__ +template<typename T> +auto ZenArrayCountHelper(T& t) -> typename std::enable_if<__is_array(T), char (&)[sizeof(t) / sizeof(t[0]) + 1]>::type; +#else +template<typename T, uint32_t N> +char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; +#endif + +#define ZEN_ARRAY_COUNT(array) (sizeof(ZenArrayCountHelper(array)) - 1) + +////////////////////////////////////////////////////////////////////////// + +#if ZEN_COMPILER_MSC +# define ZEN_NOINLINE __declspec(noinline) +#else +# define ZEN_NOINLINE __attribute__((noinline)) +#endif + +#if ZEN_PLATFORM_WINDOWS +# define ZEN_EXE_SUFFIX_LITERAL ".exe" +#else +# define ZEN_EXE_SUFFIX_LITERAL +#endif + +#define ZEN_UNUSED(...) ((void)__VA_ARGS__) +#define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__) +#define ZENCORE_API // Placeholder to allow DLL configs in the future (maybe) + +namespace zen { + +ZENCORE_API bool IsApplicationExitRequested(); +ZENCORE_API void RequestApplicationExit(int ExitCode); +ZENCORE_API bool IsDebuggerPresent(); +ZENCORE_API void SetIsInteractiveSession(bool Value); +ZENCORE_API bool IsInteractiveSession(); + +ZENCORE_API void zencore_forcelinktests(); + +} // namespace zen + +////////////////////////////////////////////////////////////////////////// + +#ifndef ZEN_USE_MIMALLOC +# if ZEN_ARCH_ARM64 + // The vcpkg mimalloc port doesn't support Arm targets +# define ZEN_USE_MIMALLOC 0 +# else +# define ZEN_USE_MIMALLOC 1 +# endif +#endif + +////////////////////////////////////////////////////////////////////////// + +#if ZEN_COMPILER_MSC +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on)) +#elif ZEN_COMPILER_GCC +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("GCC push_options") _Pragma("GCC optimize (\"O0\")") +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("GCC pop_options") +#elif ZEN_COMPILER_CLANG +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize off") +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize on") +#endif + +// Set up optimization control macros, now that we have both the build settings and the platform macros +#define ZEN_DISABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL + +#if ZEN_BUILD_DEBUG +# define ZEN_ENABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL +#else +# define ZEN_ENABLE_OPTIMIZATION ZEN_ENABLE_OPTIMIZATION_ACTUAL +#endif + +#define ZEN_ENABLE_OPTIMIZATION_ALWAYS ZEN_ENABLE_OPTIMIZATION_ACTUAL + +////////////////////////////////////////////////////////////////////////// + +#ifndef ZEN_WITH_TRACE +# define ZEN_WITH_TRACE 0 +#endif + +////////////////////////////////////////////////////////////////////////// + +using ThreadId_t = uint32_t; |