diff options
| author | Stefan Boberg <[email protected]> | 2021-05-11 13:05:39 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2021-05-11 13:05:39 +0200 |
| commit | f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa (patch) | |
| tree | 1daf7621e110d48acd5e12e3073ce48ef0dd11b2 /zencore/include | |
| download | zen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.tar.xz zen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.zip | |
Adding zenservice code
Diffstat (limited to 'zencore/include')
37 files changed, 6504 insertions, 0 deletions
diff --git a/zencore/include/zencore/atomic.h b/zencore/include/zencore/atomic.h new file mode 100644 index 000000000..457128bd4 --- /dev/null +++ b/zencore/include/zencore/atomic.h @@ -0,0 +1,43 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <intrin.h> +#include <cinttypes> + +namespace zen { + +inline uint32_t +AtomicIncrement(volatile uint32_t& value) +{ + return _InterlockedIncrement((long volatile*)&value); +} +inline uint32_t +AtomicDecrement(volatile uint32_t& value) +{ + return _InterlockedDecrement((long volatile*)&value); +} + +inline uint64_t +AtomicIncrement(volatile uint64_t& value) +{ + return _InterlockedIncrement64((__int64 volatile*)&value); +} +inline uint64_t +AtomicDecrement(volatile uint64_t& value) +{ + return _InterlockedDecrement64((__int64 volatile*)&value); +} + +inline uint32_t +AtomicAdd(volatile uint32_t& value, uint32_t amount) +{ + return _InterlockedExchangeAdd((long volatile*)&value, amount); +} +inline uint64_t +AtomicAdd(volatile uint64_t& value, uint64_t amount) +{ + return _InterlockedExchangeAdd64((__int64 volatile*)&value, amount); +} + +} // namespace zen diff --git a/zencore/include/zencore/blake3.h b/zencore/include/zencore/blake3.h new file mode 100644 index 000000000..1ef921c30 --- /dev/null +++ b/zencore/include/zencore/blake3.h @@ -0,0 +1,57 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <cinttypes> +#include <compare> +#include <cstring> + +namespace zen { + +class StringBuilderBase; + +/** + * BLAKE3 hash - 256 bits + */ +struct BLAKE3 +{ + uint8_t Hash[32]; + + inline auto operator<=>(const BLAKE3& rhs) const = default; + + static BLAKE3 HashMemory(const void* data, size_t byteCount); + static BLAKE3 FromHexString(const char* string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static const int StringLength = 64; + typedef char String_t[StringLength + 1]; + + static BLAKE3 Zero; // Initialized to all zeroes + + struct Hasher + { + size_t operator()(const BLAKE3& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct BLAKE3Stream +{ + BLAKE3Stream(); + + void Reset(); /// Begin streaming hash compute (not needed on freshly constructed instance) + BLAKE3Stream& Append(const void* data, size_t byteCount); /// Append another chunk + BLAKE3 GetHash(); /// Obtain final hash. If you wish to reuse the instance call reset() + +private: + alignas(16) uint8_t m_HashState[2048]; +}; + +void blake3_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h new file mode 100644 index 000000000..c2d276c21 --- /dev/null +++ b/zencore/include/zencore/compactbinary.h @@ -0,0 +1,1335 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/enumflags.h> +#include <zencore/intmath.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/memory.h> +#include <zencore/meta.h> +#include <zencore/sharedbuffer.h> +#include <zencore/uid.h> +#include <zencore/varint.h> + +#include <functional> +#include <memory> +#include <string> +#include <string_view> +#include <type_traits> +#include <vector> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +class CbObjectView; +class CbArrayView; +class BinaryReader; +class BinaryWriter; + +class DateTime +{ +public: + explicit DateTime(uint64_t InTicks) : Ticks(InTicks) {} + inline DateTime(int Year, int Month, int Day, int Hours = 0, int Minutes = 0, int Seconds = 0, int MilliSeconds = 0) + { + Set(Year, Month, Day, Hours, Minutes, Seconds, MilliSeconds); + } + + inline uint64_t GetTicks() const { return Ticks; } + inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; } + inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; } + +private: + void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond); + uint64_t Ticks; +}; + +class TimeSpan +{ +public: + explicit TimeSpan(uint64_t InTicks) : Ticks(InTicks) {} + inline TimeSpan(int Hours, int Minutes, int Seconds) { Set(0, Hours, Minutes, Seconds, 0); } + inline TimeSpan(int Days, int Hours, int Minutes, int Seconds) { Set(Days, Hours, Minutes, Seconds, 0); } + inline TimeSpan(int Days, int Hours, int Minutes, int Seconds, int Nanos) { Set(Days, Hours, Minutes, Seconds, Nanos); } + + inline uint64_t GetTicks() const { return Ticks; } + inline bool operator==(const TimeSpan& Rhs) const { return Ticks == Rhs.Ticks; } + inline auto operator<=>(const TimeSpan& Rhs) const { return Ticks - Rhs.Ticks; } + + /** + * Time span related constants. + */ + + /** The maximum number of ticks that can be represented in FTimespan. */ + static constexpr int64_t MaxTicks = 9223372036854775807; + + /** The minimum number of ticks that can be represented in FTimespan. */ + static constexpr int64_t MinTicks = -9223372036854775807 - 1; + + /** The number of nanoseconds per tick. */ + static constexpr int64_t NanosecondsPerTick = 100; + + /** The number of timespan ticks per day. */ + static constexpr int64_t TicksPerDay = 864000000000; + + /** The number of timespan ticks per hour. */ + static constexpr int64_t TicksPerHour = 36000000000; + + /** The number of timespan ticks per microsecond. */ + static constexpr int64_t TicksPerMicrosecond = 10; + + /** The number of timespan ticks per millisecond. */ + static constexpr int64_t TicksPerMillisecond = 10000; + + /** The number of timespan ticks per minute. */ + static constexpr int64_t TicksPerMinute = 600000000; + + /** The number of timespan ticks per second. */ + static constexpr int64_t TicksPerSecond = 10000000; + + /** The number of timespan ticks per week. */ + static constexpr int64_t TicksPerWeek = 6048000000000; + + /** The number of timespan ticks per year (365 days, not accounting for leap years). */ + static constexpr int64_t TicksPerYear = 365 * TicksPerDay; + +private: + void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano); + + uint64_t Ticks; +}; + +struct Guid +{ + uint32_t A, B, C, D; +}; + +////////////////////////////////////////////////////////////////////////// + +/** + * Field types and flags for CbField. + * + * This is a private type and is only declared here to enable inline use below. + * + * DO NOT CHANGE THE VALUE OF ANY MEMBERS OF THIS ENUM! + * BACKWARD COMPATIBILITY REQUIRES THAT THESE VALUES BE FIXED! + * SERIALIZATION USES HARD-CODED CONSTANTS BASED ON THESE VALUES! + */ +enum class CbFieldType : uint8_t +{ + /** A field type that does not occur in a valid object. */ + None = 0x00, + + /** Null. Payload is empty. */ + Null = 0x01, + + /** + * Object is an array of fields with unique non-empty names. + * + * Payload is a VarUInt byte count for the encoded fields followed by the fields. + */ + Object = 0x02, + /** + * UniformObject is an array of fields with the same field types and unique non-empty names. + * + * Payload is a VarUInt byte count for the encoded fields followed by the fields. + */ + UniformObject = 0x03, + + /** + * Array is an array of fields with no name that may be of different types. + * + * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by the fields. + */ + Array = 0x04, + /** + * UniformArray is an array of fields with no name and with the same field type. + * + * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by field type, + * followed by the fields without their field type. + */ + UniformArray = 0x05, + + /** Binary. Payload is a VarUInt byte count followed by the data. */ + Binary = 0x06, + + /** String in UTF-8. Payload is a VarUInt byte count then an unterminated UTF-8 string. */ + String = 0x07, + + /** + * Non-negative integer with the range of a 64-bit unsigned integer. + * + * Payload is the value encoded as a VarUInt. + */ + IntegerPositive = 0x08, + /** + * Negative integer with the range of a 64-bit signed integer. + * + * Payload is the ones' complement of the value encoded as a VarUInt. + */ + IntegerNegative = 0x09, + + /** Single precision float. Payload is one big endian IEEE 754 binary32 float. */ + Float32 = 0x0a, + /** Double precision float. Payload is one big endian IEEE 754 binary64 float. */ + Float64 = 0x0b, + + /** Boolean false value. Payload is empty. */ + BoolFalse = 0x0c, + /** Boolean true value. Payload is empty. */ + BoolTrue = 0x0d, + + /** + * CompactBinaryAttachment is a reference to a compact binary attachment stored externally. + * + * Payload is a 160-bit hash digest of the referenced compact binary data. + */ + CompactBinaryAttachment = 0x0e, + /** + * BinaryAttachment is a reference to a binary attachment stored externally. + * + * Payload is a 160-bit hash digest of the referenced binary data. + */ + BinaryAttachment = 0x0f, + + /** Hash. Payload is a 160-bit hash digest. */ + Hash = 0x10, + /** UUID/GUID. Payload is a 128-bit UUID as defined by RFC 4122. */ + Uuid = 0x11, + + /** + * Date and time between 0001-01-01 00:00:00.0000000 and 9999-12-31 23:59:59.9999999. + * + * Payload is a big endian int64 count of 100ns ticks since 0001-01-01 00:00:00.0000000. + */ + DateTime = 0x12, + /** + * Difference between two date/time values. + * + * Payload is a big endian int64 count of 100ns ticks in the span, and may be negative. + */ + TimeSpan = 0x13, + + /** + * Object ID + * + * Payload is a 12-byte opaque identifier + */ + ObjectId = 0x14, + + /** + * CustomById identifies the sub-type of its payload by an integer identifier. + * + * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed + * by a VarUInt of the sub-type identifier then the payload of the sub-type. + */ + CustomById = 0x1e, + /** + * CustomByType identifies the sub-type of its payload by a string identifier. + * + * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed + * by a VarUInt byte count of the unterminated sub-type identifier, then the sub-type identifier + * without termination, then the payload of the sub-type. + */ + CustomByName = 0x1f, + + /** Reserved for future use as a flag. Do not add types in this range. */ + Reserved = 0x20, + + /** + * A transient flag which indicates that the object or array containing this field has stored + * the field type before the payload and name. Non-uniform objects and fields will set this. + * + * Note: Since the flag must never be serialized, this bit may be repurposed in the future. + */ + HasFieldType = 0x40, + + /** A persisted flag which indicates that the field has a name stored before the payload. */ + HasFieldName = 0x80, +}; + +ENUM_CLASS_FLAGS(CbFieldType); + +/** Functions that operate on CbFieldType. */ +class CbFieldTypeOps +{ + static constexpr CbFieldType SerializedTypeMask = CbFieldType(0b1011'1111); + static constexpr CbFieldType TypeMask = CbFieldType(0b0011'1111); + static constexpr CbFieldType ObjectMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType ObjectBase = CbFieldType(0b0000'0010); + static constexpr CbFieldType ArrayMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType ArrayBase = CbFieldType(0b0000'0100); + static constexpr CbFieldType IntegerMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType IntegerBase = CbFieldType(0b0000'1000); + static constexpr CbFieldType FloatMask = CbFieldType(0b0011'1100); + static constexpr CbFieldType FloatBase = CbFieldType(0b0000'1000); + static constexpr CbFieldType BoolMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType BoolBase = CbFieldType(0b0000'1100); + static constexpr CbFieldType AttachmentMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType AttachmentBase = CbFieldType(0b0000'1110); + + static void StaticAssertTypeConstants(); + +public: + /** The type with flags removed. */ + static constexpr inline CbFieldType GetType(CbFieldType Type) { return Type & TypeMask; } + /** The type with transient flags removed. */ + static constexpr inline CbFieldType GetSerializedType(CbFieldType Type) { return Type & SerializedTypeMask; } + + static constexpr inline bool HasFieldType(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldType); } + static constexpr inline bool HasFieldName(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldName); } + + static constexpr inline bool IsNone(CbFieldType Type) { return GetType(Type) == CbFieldType::None; } + static constexpr inline bool IsNull(CbFieldType Type) { return GetType(Type) == CbFieldType::Null; } + + static constexpr inline bool IsObject(CbFieldType Type) { return (Type & ObjectMask) == ObjectBase; } + static constexpr inline bool IsArray(CbFieldType Type) { return (Type & ArrayMask) == ArrayBase; } + + static constexpr inline bool IsBinary(CbFieldType Type) { return GetType(Type) == CbFieldType::Binary; } + static constexpr inline bool IsString(CbFieldType Type) { return GetType(Type) == CbFieldType::String; } + + static constexpr inline bool IsInteger(CbFieldType Type) { return (Type & IntegerMask) == IntegerBase; } + /** Whether the field is a float, or integer due to implicit conversion. */ + static constexpr inline bool IsFloat(CbFieldType Type) { return (Type & FloatMask) == FloatBase; } + static constexpr inline bool IsBool(CbFieldType Type) { return (Type & BoolMask) == BoolBase; } + + static constexpr inline bool IsCompactBinaryAttachment(CbFieldType Type) + { + return GetType(Type) == CbFieldType::CompactBinaryAttachment; + } + static constexpr inline bool IsBinaryAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::BinaryAttachment; } + static constexpr inline bool IsAttachment(CbFieldType Type) { return (Type & AttachmentMask) == AttachmentBase; } + + static constexpr inline bool IsHash(CbFieldType Type) + { + switch (GetType(Type)) + { + case CbFieldType::Hash: + case CbFieldType::BinaryAttachment: + case CbFieldType::CompactBinaryAttachment: + return true; + default: + return false; + } + } + + static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; } + static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; } + + static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; } + static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; } + + /** Whether the type is or may contain fields of any attachment type. */ + static constexpr inline bool MayContainAttachments(CbFieldType Type) + { + // The use of !! will suppress V792 from static analysis. Using //-V792 did not work. + return !!IsObject(Type) | !!IsArray(Type) | !!IsAttachment(Type); + } +}; + +/** Errors that can occur when accessing a field. */ +enum class CbFieldError : uint8_t +{ + /** The field is not in an error state. */ + None, + /** The value type does not match the requested type. */ + TypeError, + /** The value is out of range for the requested type. */ + RangeError, +}; + +class ICbVisitor +{ +public: + virtual void SetName(std::string_view Name) = 0; + virtual void BeginObject() = 0; + virtual void EndObject() = 0; + virtual void BeginArray() = 0; + virtual void EndArray() = 0; + virtual void VisitNull() = 0; + virtual void VisitBinary(SharedBuffer Value) = 0; + virtual void VisitString(std::string_view Value) = 0; + virtual void VisitInteger(int64_t Value) = 0; + virtual void VisitInteger(uint64_t Value) = 0; + virtual void VisitFloat(float Value) = 0; + virtual void VisitDouble(double Value) = 0; + virtual void VisitBool(bool value) = 0; + virtual void VisitCbAttachment(const IoHash& Value) = 0; + virtual void VisitBinaryAttachment(const IoHash& Value) = 0; + virtual void VisitHash(const IoHash& Value) = 0; + virtual void VisitUuid(const Guid& Value) = 0; + virtual void VisitObjectId(const Oid& Value) = 0; + virtual void VisitDateTime(DateTime Value) = 0; + virtual void VisitTimeSpan(TimeSpan Value) = 0; +}; + +/** + * An atom of data in the compact binary format. + * + * Accessing the value of a field is always a safe operation, even if accessed as the wrong type. + * An invalid access will return a default value for the requested type, and set an error code on + * the field that can be checked with GetLastError and HasLastError. A valid access will clear an + * error from a previous invalid access. + * + * A field is encoded in one or more bytes, depending on its type and the type of object or array + * that contains it. A field of an object or array which is non-uniform encodes its field type in + * the first byte, and includes the HasFieldName flag for a field in an object. The field name is + * encoded in a variable-length unsigned integer of its size in bytes, for named fields, followed + * by that many bytes of the UTF-8 encoding of the name with no null terminator. The remainder of + * the field is the payload and is described in the field type enum. Every field must be uniquely + * addressable when encoded, which means a zero-byte field is not permitted, and only arises in a + * uniform array of fields with no payload, where the answer is to encode as a non-uniform array. + * + * This type only provides a view into memory and does not perform any memory management itself. + * Use CbFieldRef to hold a reference to the underlying memory when necessary. + */ + +class CbFieldView +{ +public: + CbFieldView() = default; + + ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType); + + /** Returns the name of the field if it has a name, otherwise an empty view. */ + constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); } + + ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView()); + ZENCORE_API CbObjectView AsObjectView(); + ZENCORE_API CbArrayView AsArrayView(); + ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view()); + + ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const; + + /** Access the field as an int8. Returns the provided default on error. */ + inline int8_t AsInt8(int8_t Default = 0) { return AsInteger<int8_t>(Default); } + /** Access the field as an int16. Returns the provided default on error. */ + inline int16_t AsInt16(int16_t Default = 0) { return AsInteger<int16_t>(Default); } + /** Access the field as an int32. Returns the provided default on error. */ + inline int32_t AsInt32(int32_t Default = 0) { return AsInteger<int32_t>(Default); } + /** Access the field as an int64. Returns the provided default on error. */ + inline int64_t AsInt64(int64_t Default = 0) { return AsInteger<int64_t>(Default); } + /** Access the field as a uint8. Returns the provided default on error. */ + inline uint8_t AsUInt8(uint8_t Default = 0) { return AsInteger<uint8_t>(Default); } + /** Access the field as a uint16. Returns the provided default on error. */ + inline uint16_t AsUInt16(uint16_t Default = 0) { return AsInteger<uint16_t>(Default); } + /** Access the field as a uint32. Returns the provided default on error. */ + inline uint32_t AsUInt32(uint32_t Default = 0) { return AsInteger<uint32_t>(Default); } + /** Access the field as a uint64. Returns the provided default on error. */ + inline uint64_t AsUInt64(uint64_t Default = 0) { return AsInteger<uint64_t>(Default); } + + /** Access the field as a float. Returns the provided default on error. */ + ZENCORE_API float AsFloat(float Default = 0.0f); + /** Access the field as a double. Returns the provided default on error. */ + ZENCORE_API double AsDouble(double Default = 0.0); + + /** Access the field as a bool. Returns the provided default on error. */ + ZENCORE_API bool AsBool(bool bDefault = false); + + /** Access the field as a hash referencing a compact binary attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsCompactBinaryAttachment(const IoHash& Default = IoHash()); + /** Access the field as a hash referencing a binary attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsBinaryAttachment(const IoHash& Default = IoHash()); + /** Access the field as a hash referencing an attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsAttachment(const IoHash& Default = IoHash()); + + /** Access the field as a hash. Returns the provided default on error. */ + ZENCORE_API IoHash AsHash(const IoHash& Default = IoHash()); + + /** Access the field as a UUID. Returns a nil UUID on error. */ + ZENCORE_API Guid AsUuid(); + /** Access the field as a UUID. Returns the provided default on error. */ + ZENCORE_API Guid AsUuid(const Guid& Default); + + /** Access the field as an OID. Returns a nil OID on error. */ + ZENCORE_API Oid AsObjectId(); + /** Access the field as a OID. Returns the provided default on error. */ + ZENCORE_API Oid AsObjectId(const Oid& Default); + + /** Access the field as a date/time tick count. Returns the provided default on error. */ + ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0); + + /** Access the field as a date/time. Returns a date/time at the epoch on error. */ + ZENCORE_API DateTime AsDateTime(); + /** Access the field as a date/time. Returns the provided default on error. */ + ZENCORE_API DateTime AsDateTime(DateTime Default); + + /** Access the field as a timespan tick count. Returns the provided default on error. */ + ZENCORE_API int64_t AsTimeSpanTicks(int64_t Default = 0); + + /** Access the field as a timespan. Returns an empty timespan on error. */ + ZENCORE_API TimeSpan AsTimeSpan(); + /** Access the field as a timespan. Returns the provided default on error. */ + ZENCORE_API TimeSpan AsTimeSpan(TimeSpan Default); + + /** True if the field has a name. */ + constexpr inline bool HasName() const { return CbFieldTypeOps::HasFieldName(Type); } + + constexpr inline bool IsNull() const { return CbFieldTypeOps::IsNull(Type); } + + constexpr inline bool IsObject() const { return CbFieldTypeOps::IsObject(Type); } + constexpr inline bool IsArray() const { return CbFieldTypeOps::IsArray(Type); } + + constexpr inline bool IsBinary() const { return CbFieldTypeOps::IsBinary(Type); } + constexpr inline bool IsString() const { return CbFieldTypeOps::IsString(Type); } + + /** Whether the field is an integer of unspecified range and sign. */ + constexpr inline bool IsInteger() const { return CbFieldTypeOps::IsInteger(Type); } + /** Whether the field is a float, or integer that supports implicit conversion. */ + constexpr inline bool IsFloat() const { return CbFieldTypeOps::IsFloat(Type); } + constexpr inline bool IsBool() const { return CbFieldTypeOps::IsBool(Type); } + + constexpr inline bool IsCompactBinaryAttachment() const { return CbFieldTypeOps::IsCompactBinaryAttachment(Type); } + constexpr inline bool IsBinaryAttachment() const { return CbFieldTypeOps::IsBinaryAttachment(Type); } + constexpr inline bool IsAttachment() const { return CbFieldTypeOps::IsAttachment(Type); } + + constexpr inline bool IsHash() const { return CbFieldTypeOps::IsHash(Type); } + constexpr inline bool IsUuid() const { return CbFieldTypeOps::IsUuid(Type); } + constexpr inline bool IsObjectId() const { return CbFieldTypeOps::IsObjectId(Type); } + + constexpr inline bool IsDateTime() const { return CbFieldTypeOps::IsDateTime(Type); } + constexpr inline bool IsTimeSpan() const { return CbFieldTypeOps::IsTimeSpan(Type); } + + /** Whether the field has a value. */ + constexpr inline explicit operator bool() const { return HasValue(); } + + /** + * Whether the field has a value. + * + * All fields in a valid object or array have a value. A field with no value is returned when + * finding a field by name fails or when accessing an iterator past the end. + */ + constexpr inline bool HasValue() const { return !CbFieldTypeOps::IsNone(Type); }; + + /** Whether the last field access encountered an error. */ + constexpr inline bool HasError() const { return Error != CbFieldError::None; } + + /** The type of error that occurred on the last field access, or None. */ + constexpr inline CbFieldError GetError() const { return Error; } + + /** Returns the size of the field in bytes, including the type and name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the field, including the type and name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& HashStream) const; + + /** Feed the field (including type and name) to the stream function */ + inline void WriteToStream(auto Hash) const + { + const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type); + Hash(&SerializedType, sizeof(SerializedType)); + auto View = GetViewNoType(); + Hash(View.GetData(), View.GetSize()); + } + + /** Copy the field into a buffer of exactly GetSize() bytes, including the type and name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the field into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + /** + * Whether this field is identical to the other field. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be performed with ValidateCompactBinary, except for field order and field name case. + */ + ZENCORE_API bool Equals(const CbFieldView& Other) const; + + /** Returns a view of the field, including the type and name when present. */ + ZENCORE_API MemoryView GetView() const; + + /** + * Try to get a view of the field as it would be serialized, such as by CopyTo. + * + * A serialized view is not available if the field has an externally-provided type. + * Access the serialized form of such fields using CopyTo or FCbFieldRef::Clone. + */ + inline bool TryGetSerializedView(MemoryView& OutView) const + { + if (CbFieldTypeOps::HasFieldType(Type)) + { + OutView = GetView(); + return true; + } + return false; + } + +protected: + /** Returns a view of the name and value payload, which excludes the type. */ + ZENCORE_API MemoryView GetViewNoType() const; + + /** Returns a view of the value payload, which excludes the type and name. */ + inline MemoryView GetPayloadView() const { return MemoryView(Payload, GetPayloadSize()); } + + /** Returns the type of the field including flags. */ + constexpr inline CbFieldType GetType() const { return Type; } + + /** Returns the start of the value payload. */ + constexpr inline const void* GetPayload() const { return Payload; } + + /** Returns the end of the value payload. */ + inline const void* GetPayloadEnd() const { return static_cast<const uint8_t*>(Payload) + GetPayloadSize(); } + + /** Returns the size of the value payload in bytes, which is the field excluding the type and name. */ + ZENCORE_API uint64_t GetPayloadSize() const; + + /** Assign a field from a pointer to its data and an optional externally-provided type. */ + inline void Assign(const void* InData, const CbFieldType InType) + { + static_assert(std::is_trivially_destructible<CbFieldView>::value, + "This optimization requires CbField to be trivially destructible!"); + new (this) CbFieldView(InData, InType); + } + +private: + /** Parameters for converting to an integer. */ + struct IntegerParams + { + /** Whether the output type has a sign bit. */ + uint32_t IsSigned : 1; + /** Bits of magnitude. (7 for int8) */ + uint32_t MagnitudeBits : 31; + }; + + /** Make integer params for the given integer type. */ + template<typename IntType> + static constexpr inline IntegerParams MakeIntegerParams() + { + IntegerParams Params; + Params.IsSigned = IntType(-1) < IntType(0); + Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned; + return Params; + } + + /** + * Access the field as the given integer type. + * + * Returns the provided default if the value cannot be represented in the output type. + */ + template<typename IntType> + inline IntType AsInteger(IntType Default) + { + return IntType(AsInteger(uint64_t(Default), MakeIntegerParams<IntType>())); + } + + ZENCORE_API uint64_t AsInteger(uint64_t Default, IntegerParams Params); + + /** The field type, with the transient HasFieldType flag if the field contains its type. */ + CbFieldType Type = CbFieldType::None; + /** The error (if any) that occurred on the last field access. */ + CbFieldError Error = CbFieldError::None; + /** The number of bytes for the name stored before the payload. */ + uint32_t NameLen = 0; + /** The value payload, which also points to the end of the name. */ + const void* Payload = nullptr; +}; + +template<typename FieldType> +class TCbFieldIterator : public FieldType +{ +public: + /** Construct an empty field range. */ + constexpr TCbFieldIterator() = default; + + inline TCbFieldIterator& operator++() + { + const void* const PayloadEnd = FieldType::GetPayloadEnd(); + const int64_t AtEndMask = int64_t(PayloadEnd == FieldsEnd) - 1; + const CbFieldType NextType = CbFieldType(int64_t(FieldType::GetType()) & AtEndMask); + const void* const NextField = reinterpret_cast<const void*>(int64_t(PayloadEnd) & AtEndMask); + const void* const NextFieldsEnd = reinterpret_cast<const void*>(int64_t(FieldsEnd) & AtEndMask); + + FieldType::Assign(NextField, NextType); + FieldsEnd = NextFieldsEnd; + return *this; + } + + inline TCbFieldIterator operator++(int) + { + TCbFieldIterator It(*this); + ++*this; + return It; + } + + constexpr inline FieldType& operator*() { return *this; } + constexpr inline FieldType* operator->() { return this; } + + /** Reset this to an empty field range. */ + inline void Reset() { *this = TCbFieldIterator(); } + + /** Returns the size of the fields in the range in bytes. */ + ZENCORE_API uint64_t GetRangeSize() const; + + /** Calculate the hash of every field in the range. */ + ZENCORE_API IoHash GetRangeHash() const; + ZENCORE_API void GetRangeHash(IoHashStream& Hash) const; + + using FieldType::Equals; + + template<typename OtherFieldType> + constexpr inline bool Equals(const TCbFieldIterator<OtherFieldType>& Other) const + { + return FieldType::GetPayload() == Other.OtherFieldType::GetPayload() && FieldsEnd == Other.FieldsEnd; + } + + template<typename OtherFieldType> + constexpr inline bool operator==(const TCbFieldIterator<OtherFieldType>& Other) const + { + return Equals(Other); + } + + template<typename OtherFieldType> + constexpr inline bool operator!=(const TCbFieldIterator<OtherFieldType>& Other) const + { + return !Equals(Other); + } + + /** Copy the field range into a buffer of exactly GetRangeSize() bytes. */ + ZENCORE_API void CopyRangeTo(MutableMemoryView Buffer) const; + + /** Invoke the visitor for every attachment in the field range. */ + ZENCORE_API void IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const; + + /** Create a view of every field in the range. */ + inline MemoryView GetRangeView() const { return MemoryView(FieldType::GetView().GetData(), FieldsEnd); } + + /** + * Try to get a view of every field in the range as they would be serialized. + * + * A serialized view is not available if the underlying fields have an externally-provided type. + * Access the serialized form of such ranges using CbFieldRefIterator::CloneRange. + */ + inline bool TryGetSerializedRangeView(MemoryView& OutView) const + { + if (CbFieldTypeOps::HasFieldType(FieldType::GetType())) + { + OutView = GetRangeView(); + return true; + } + return false; + } + +protected: + /** Construct a field range that contains exactly one field. */ + constexpr inline explicit TCbFieldIterator(FieldType InField) : FieldType(std::move(InField)), FieldsEnd(FieldType::GetPayloadEnd()) {} + + /** + * Construct a field range from the first field and a pointer to the end of the last field. + * + * @param InField The first field, or the default field if there are no fields. + * @param InFieldsEnd A pointer to the end of the payload of the last field, or null. + */ + constexpr inline TCbFieldIterator(FieldType&& InField, const void* InFieldsEnd) : FieldType(std::move(InField)), FieldsEnd(InFieldsEnd) + { + } + + /** Returns the end of the last field, or null for an iterator at the end. */ + template<typename OtherFieldType> + static inline const void* GetFieldsEnd(const TCbFieldIterator<OtherFieldType>& It) + { + return It.FieldsEnd; + } + +private: + friend inline TCbFieldIterator begin(const TCbFieldIterator& Iterator) { return Iterator; } + friend inline TCbFieldIterator end(const TCbFieldIterator&) { return TCbFieldIterator(); } + +private: + template<typename OtherType> + friend class TCbFieldIterator; + + friend class CbFieldViewIterator; + + /** Pointer to the first byte past the end of the last field. Set to null at the end. */ + const void* FieldsEnd = nullptr; +}; + +/** + * Iterator for CbField. + * + * @see CbFieldIterator + */ +class CbFieldViewIterator : public TCbFieldIterator<CbFieldView> +{ +public: + constexpr CbFieldViewIterator() = default; + + /** Construct a field range that contains exactly one field. */ + static inline CbFieldViewIterator MakeSingle(const CbFieldView& Field) { return CbFieldViewIterator(Field); } + + /** + * Construct a field range from a buffer containing zero or more valid fields. + * + * @param View A buffer containing zero or more valid fields. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + */ + static inline CbFieldViewIterator MakeRange(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType) + { + return !View.IsEmpty() ? TCbFieldIterator(CbFieldView(View.GetData(), Type), View.GetDataEnd()) : CbFieldViewIterator(); + } + + /** Construct an iterator from another iterator. */ + template<typename OtherFieldType> + inline CbFieldViewIterator(const TCbFieldIterator<OtherFieldType>& It) + : TCbFieldIterator(ImplicitConv<CbFieldView>(It), GetFieldsEnd(It)) + { + } + +private: + using TCbFieldIterator::TCbFieldIterator; +}; + +/** + * Array of CbField that have no names. + * + * Accessing a field of the array requires iteration. Access by index is not provided because the + * cost of accessing an item by index scales linearly with the index. + * + * This type only provides a view into memory and does not perform any memory management itself. + * Use CbArrayRef to hold a reference to the underlying memory when necessary. + */ +class CbArrayView : protected CbFieldView +{ + friend class CbFieldView; + +public: + /** @see CbField::CbField */ + using CbFieldView::CbFieldView; + + /** Construct an array with no fields. */ + ZENCORE_API CbArrayView(); + + /** Returns the number of items in the array. */ + ZENCORE_API uint64_t Num() const; + + /** Create an iterator for the fields of this array. */ + ZENCORE_API CbFieldViewIterator CreateViewIterator() const; + + /** Visit the fields of this array. */ + ZENCORE_API void VisitFields(ICbVisitor& Visitor); + + /** Access the array as an array field. */ + inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } + + /** Construct an array from an array field. No type check is performed! */ + static inline CbArrayView FromFieldView(const CbFieldView& Field) { return CbArrayView(Field); } + + /** Returns the size of the array in bytes if serialized by itself with no name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the array if serialized by itself with no name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& Stream) const; + + /** + * Whether this array is identical to the other array. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be done with the All mode to check these assumptions about the format of the inputs. + */ + ZENCORE_API bool Equals(const CbArrayView& Other) const; + + /** Copy the array into a buffer of exactly GetSize() bytes, with no name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the array into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + ///** Invoke the visitor for every attachment in the array. */ + inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const + { + CreateViewIterator().IterateRangeAttachments(Visitor); + } + + /** Returns a view of the array, including the type and name when present. */ + using CbFieldView::GetView; + +private: + friend inline CbFieldViewIterator begin(const CbArrayView& Array) { return Array.CreateViewIterator(); } + friend inline CbFieldViewIterator end(const CbArrayView&) { return CbFieldViewIterator(); } + + /** Construct an array from an array field. No type check is performed! Use via FromField. */ + inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {} +}; + +class CbObjectView : protected CbFieldView +{ + friend class CbFieldView; + +public: + /** @see CbField::CbField */ + using CbFieldView::CbFieldView; + + /** Construct an object with no fields. */ + ZENCORE_API CbObjectView(); + + /** Create an iterator for the fields of this object. */ + ZENCORE_API CbFieldViewIterator CreateViewIterator() const; + + /** Visit the fields of this object. */ + ZENCORE_API void VisitFields(ICbVisitor& Visitor); + + /** + * Find a field by case-sensitive name comparison. + * + * The cost of this operation scales linearly with the number of fields in the object. Prefer + * to iterate over the fields only once when consuming an object. + * + * @param Name The name of the field. + * @return The matching field if found, otherwise a field with no value. + */ + ZENCORE_API CbFieldView FindView(std::string_view Name) const; + + /** Find a field by case-insensitive name comparison. */ + ZENCORE_API CbFieldView FindViewIgnoreCase(std::string_view Name) const; + + /** Find a field by case-sensitive name comparison. */ + inline CbFieldView operator[](std::string_view Name) const { return FindView(Name); } + + /** Access the object as an object field. */ + inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } + + /** Construct an object from an object field. No type check is performed! */ + static inline CbObjectView FromFieldView(const CbFieldView& Field) { return CbObjectView(Field); } + + /** Returns the size of the object in bytes if serialized by itself with no name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the object if serialized by itself with no name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& HashStream) const; + + /** + * Whether this object is identical to the other object. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be done with the All mode to check these assumptions about the format of the inputs. + */ + ZENCORE_API bool Equals(const CbObjectView& Other) const; + + /** Copy the object into a buffer of exactly GetSize() bytes, with no name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the field into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + ///** Invoke the visitor for every attachment in the object. */ + inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const + { + CreateViewIterator().IterateRangeAttachments(Visitor); + } + + /** Returns a view of the object, including the type and name when present. */ + using CbFieldView::GetView; + + /** Whether the field has a value. */ + using CbFieldView::operator bool; + +private: + friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); } + friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); } + + /** Construct an object from an object field. No type check is performed! Use via FromField. */ + inline explicit CbObjectView(const CbFieldView& Field) : CbFieldView(Field) {} +}; + +////////////////////////////////////////////////////////////////////////// + +/** A reference to a function that is used to allocate buffers for compact binary data. */ +using BufferAllocator = std::function<UniqueBuffer(uint64_t Size)>; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** A wrapper that holds a reference to the buffer that contains its compact binary value. */ +template<typename BaseType> +class CbBuffer : public BaseType +{ +public: + /** Construct a default value. */ + CbBuffer() = default; + + /** + * Construct a value from a pointer to its data and an optional externally-provided type. + * + * @param ValueBuffer A buffer that exactly contains the value. + * @param Type HasFieldType means that ValueBuffer contains the type. Otherwise, use the given type. + */ + inline explicit CbBuffer(SharedBuffer ValueBuffer, CbFieldType Type = CbFieldType::HasFieldType) + { + if (ValueBuffer) + { + BaseType::operator=(BaseType(ValueBuffer.GetData(), Type)); + ZEN_ASSERT(ValueBuffer.GetView().Contains(BaseType::GetView())); + Buffer = std::move(ValueBuffer); + } + } + + /** Construct a value that holds a reference to the buffer that contains it. */ + inline CbBuffer(const BaseType& Value, SharedBuffer OuterBuffer) : BaseType(Value) + { + if (OuterBuffer) + { + ZEN_ASSERT(OuterBuffer.GetView().Contains(BaseType::GetView())); + Buffer = std::move(OuterBuffer); + } + } + + /** Construct a value that holds a reference to the buffer of the outer that contains it. */ + template<typename OtherBaseType> + inline CbBuffer(const BaseType& Value, CbBuffer<OtherBaseType> OuterRef) : CbBuffer(Value, std::move(OuterRef.Buffer)) + { + } + + /** Reset this to a default value and null buffer. */ + inline void Reset() { *this = CbBuffer(); } + + /** Whether this reference has ownership of the memory in its buffer. */ + inline bool IsOwned() const { return Buffer && Buffer.IsOwned(); } + + /** Clone the value, if necessary, to a buffer that this reference has ownership of. */ + inline void MakeOwned() + { + if (!IsOwned()) + { + UniqueBuffer MutableBuffer = UniqueBuffer::Alloc(BaseType::GetSize()); + BaseType::CopyTo(MutableBuffer); + BaseType::operator=(BaseType(MutableBuffer.GetData())); + Buffer = std::move(MutableBuffer); + } + } + + /** Returns a buffer that exactly contains this value. */ + inline SharedBuffer GetBuffer() const + { + const MemoryView View = BaseType::GetView(); + const SharedBuffer& OuterBuffer = GetOuterBuffer(); + return View == OuterBuffer.GetView() ? OuterBuffer : SharedBuffer::MakeView(View, OuterBuffer); + } + + /** Returns the outer buffer (if any) that contains this value. */ + inline const SharedBuffer& GetOuterBuffer() const& { return Buffer; } + inline SharedBuffer GetOuterBuffer() && { return std::move(Buffer); } + +private: + template<typename OtherType> + friend class CbBuffer; + + SharedBuffer Buffer; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Factory functions for types derived from CbBuffer. + * + * This uses the curiously recurring template pattern to construct the correct type of reference. + * The derived type inherits from CbBufferRef and this type to expose the factory functions. + */ +template<typename RefType, typename BaseType> +class CbBufferFactory +{ +public: + /** Construct a value from an owned clone of its memory. */ + static inline RefType Clone(const void* const Data) { return Clone(BaseType(Data)); } + + /** Construct a value from an owned clone of its memory. */ + static inline RefType Clone(const BaseType& Value) + { + RefType Ref = MakeView(Value); + Ref.MakeOwned(); + return Ref; + } + + /** Construct a value from a read-only view of its memory and its optional outer buffer. */ + static inline RefType MakeView(const void* const Data, SharedBuffer OuterBuffer = SharedBuffer()) + { + return MakeView(BaseType(Data), std::move(OuterBuffer)); + } + + /** Construct a value from a read-only view of its memory and its optional outer buffer. */ + static inline RefType MakeView(const BaseType& Value, SharedBuffer OuterBuffer = SharedBuffer()) + { + return RefType(Value, std::move(OuterBuffer)); + } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class CbArray; +class CbObject; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A field that can hold a reference to the memory that contains it. + * + * @see CbBufferRef + */ +class CbField : public CbBuffer<CbFieldView>, public CbBufferFactory<CbField, CbFieldView> +{ +public: + using CbBuffer::CbBuffer; + + /** Access the field as an object. Defaults to an empty object on error. */ + inline CbObject AsObject() &; + + /** Access the field as an object. Defaults to an empty object on error. */ + inline CbObject AsObject() &&; + + /** Access the field as an array. Defaults to an empty array on error. */ + inline CbArray AsArray() &; + + /** Access the field as an array. Defaults to an empty array on error. */ + inline CbArray AsArray() &&; +}; + +/** + * Iterator for CbFieldRef. + * + * @see CbFieldIterator + */ +class CbFieldIterator : public TCbFieldIterator<CbField> +{ +public: + /** Construct a field range from an owned clone of a range. */ + ZENCORE_API static CbFieldIterator CloneRange(const CbFieldViewIterator& It); + + /** Construct a field range from an owned clone of a range. */ + static inline CbFieldIterator CloneRange(const CbFieldIterator& It) { return CloneRange(CbFieldViewIterator(It)); } + + /** Construct a field range that contains exactly one field. */ + static inline CbFieldIterator MakeSingle(CbField Field) { return CbFieldIterator(std::move(Field)); } + + /** + * Construct a field range from a buffer containing zero or more valid fields. + * + * @param Buffer A buffer containing zero or more valid fields. + * @param Type HasFieldType means that Buffer contains the type. Otherwise, use the given type. + */ + static inline CbFieldIterator MakeRange(SharedBuffer Buffer, CbFieldType Type = CbFieldType::HasFieldType) + { + if (Buffer.GetSize()) + { + const void* const DataEnd = Buffer.GetView().GetDataEnd(); + return CbFieldIterator(CbField(std::move(Buffer), Type), DataEnd); + } + return CbFieldIterator(); + } + + /** Construct a field range from an iterator and its optional outer buffer. */ + static inline CbFieldIterator MakeRangeView(const CbFieldViewIterator& It, SharedBuffer OuterBuffer = SharedBuffer()) + { + return CbFieldIterator(CbField(It, std::move(OuterBuffer)), GetFieldsEnd(It)); + } + + /** Construct an empty field range. */ + constexpr CbFieldIterator() = default; + + /** Clone the range, if necessary, to a buffer that this reference has ownership of. */ + inline void MakeRangeOwned() + { + if (!IsOwned()) + { + *this = CloneRange(*this); + } + } + + /** Returns a buffer that exactly contains the field range. */ + ZENCORE_API SharedBuffer GetRangeBuffer() const; + +private: + using TCbFieldIterator::TCbFieldIterator; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An array that can hold a reference to the memory that contains it. + * + * @see CbBuffer + */ +class CbArray : public CbBuffer<CbArrayView>, public CbBufferFactory<CbArray, CbArrayView> +{ +public: + using CbBuffer::CbBuffer; + + /** Create an iterator for the fields of this array. */ + inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); } + + /** Access the array as an array field. */ + inline CbField AsField() const& { return CbField(CbArrayView::AsFieldView(), *this); } + + /** Access the array as an array field. */ + inline CbField AsField() && { return CbField(CbArrayView::AsFieldView(), std::move(*this)); } + +private: + friend inline CbFieldIterator begin(const CbArray& Array) { return Array.CreateIterator(); } + friend inline CbFieldIterator end(const CbArray&) { return CbFieldIterator(); } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An object that can hold a reference to the memory that contains it. + * + * @see CbBuffer + */ +class CbObject : public CbBuffer<CbObjectView>, public CbBufferFactory<CbObject, CbObjectView> +{ +public: + using CbBuffer::CbBuffer; + + /** Create an iterator for the fields of this object. */ + inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); } + + /** Find a field by case-sensitive name comparison. */ + inline CbField Find(std::string_view Name) const + { + if (CbFieldView Field = FindView(Name)) + { + return CbField(Field, *this); + } + return CbField(); + } + + /** Find a field by case-insensitive name comparison. */ + inline CbField FindIgnoreCase(std::string_view Name) const + { + if (CbFieldView Field = FindIgnoreCase(Name)) + { + return CbField(Field, *this); + } + return CbField(); + } + + /** Find a field by case-sensitive name comparison. */ + inline CbFieldView operator[](std::string_view Name) const { return Find(Name); } + + /** Access the object as an object field. */ + inline CbField AsField() const& { return CbField(CbObjectView::AsFieldView(), *this); } + + /** Access the object as an object field. */ + inline CbField AsField() && { return CbField(CbObjectView::AsFieldView(), std::move(*this)); } + +private: + friend inline CbFieldIterator begin(const CbObject& Object) { return Object.CreateIterator(); } + friend inline CbFieldIterator end(const CbObject&) { return CbFieldIterator(); } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +inline CbObject +CbField::AsObject() & +{ + return IsObject() ? CbObject(AsObjectView(), *this) : CbObject(); +} + +inline CbObject +CbField::AsObject() && +{ + return IsObject() ? CbObject(AsObjectView(), std::move(*this)) : CbObject(); +} + +inline CbArray +CbField::AsArray() & +{ + return IsArray() ? CbArray(AsArrayView(), *this) : CbArray(); +} + +inline CbArray +CbField::AsArray() && +{ + return IsArray() ? CbArray(AsArrayView(), std::move(*this)) : CbArray(); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator); + +inline CbObject +LoadCompactBinaryObject(IoBuffer Payload) +{ + return CbObject{SharedBuffer::MakeView(Payload.Data(), Payload.Size())}; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Determine the size in bytes of the compact binary field at the start of the view. + * + * This may be called on an incomplete or invalid field, in which case the returned size is zero. + * A size can always be extracted from a valid field with no name if a view of at least the first + * 10 bytes is provided, regardless of field size. For fields with names, the size of view needed + * to calculate a size is at most 10 + MaxNameLen + MeasureVarUInt(MaxNameLen). + * + * This function can be used when streaming a field, for example, to determine the size of buffer + * to fill before attempting to construct a field from it. + * + * @param View A memory view that may contain the start of a field. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + */ +ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType); + +/** + * Try to determine the type and size of the compact binary field at the start of the view. + * + * This may be called on an incomplete or invalid field, in which case it will return false, with + * OutSize being 0 for invalid fields, otherwise the minimum view size necessary to make progress + * in measuring the field on the next call to this function. + * + * @note A return of true from this function does not indicate that the entire field is valid. + * + * @param InView A memory view that may contain the start of a field. + * @param OutType The type (with flags) of the field. None is written until a value is available. + * @param OutSize The total field size for a return of true, 0 for invalid fields, or the size to + * make progress in measuring the field on the next call to this function. + * @param InType HasFieldType means that InView contains the type. Otherwise, use the given type. + * @return true if the size of the field was determined, otherwise false. + */ +ZENCORE_API bool TryMeasureCompactBinary(MemoryView InView, + CbFieldType& OutType, + uint64_t& OutSize, + CbFieldType InType = CbFieldType::HasFieldType); + +inline CbFieldViewIterator +begin(CbFieldView& View) +{ + if (View.IsArray()) + { + return View.AsArrayView().CreateViewIterator(); + } + else if (View.IsObject()) + { + return View.AsObjectView().CreateViewIterator(); + } + + return CbFieldViewIterator(); +} + +inline CbFieldViewIterator +end(CbFieldView&) +{ + return CbFieldViewIterator(); +} + +void uson_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h new file mode 100644 index 000000000..83d4309f7 --- /dev/null +++ b/zencore/include/zencore/compactbinarybuilder.h @@ -0,0 +1,633 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> + +#include <zencore/enumflags.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/refcount.h> +#include <zencore/sha1.h> + +#include <atomic> +#include <memory> +#include <string> +#include <string_view> +#include <type_traits> +#include <vector> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +class CbAttachment; +class BinaryWriter; + +/** + * A writer for compact binary object, arrays, and fields. + * + * The writer produces a sequence of fields that can be saved to a provided memory buffer or into + * a new owned buffer. The typical use case is to write a single object, which can be accessed by + * calling Save().AsObjectRef() or Save(Buffer).AsObject(). + * + * The writer will assert on most incorrect usage and will always produce valid compact binary if + * provided with valid input. The writer does not check for invalid UTF-8 string encoding, object + * fields with duplicate names, or invalid compact binary being copied from another source. + * + * It is most convenient to use the streaming API for the writer, as demonstrated in the example. + * + * When writing a small amount of compact binary data, TCbWriter can be more efficient as it uses + * a fixed-size stack buffer for storage before spilling onto the heap. + * + * @see TCbWriter + * + * Example: + * + * CbObjectRef WriteObject() + * { + * CbWriter<256> Writer; + * Writer.BeginObject(); + * + * Writer << "Resize" << true; + * Writer << "MaxWidth" << 1024; + * Writer << "MaxHeight" << 1024; + * + * Writer.BeginArray(); + * Writer << "FormatA" << "FormatB" << "FormatC"; + * Writer.EndArray(); + * + * Writer.EndObject(); + * return Writer.Save().AsObjectRef(); + * } + */ +class CbWriter +{ +public: + ZENCORE_API CbWriter(); + ZENCORE_API ~CbWriter(); + + CbWriter(const CbWriter&) = delete; + CbWriter& operator=(const CbWriter&) = delete; + + /** Empty the writer without releasing any allocated memory. */ + ZENCORE_API void Reset(); + + /** + * Serialize the field(s) to an owned buffer and return it as an iterator. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + * The writer remains valid for further use when this function returns. + */ + ZENCORE_API CbFieldIterator Save(); + + /** + * Serialize the field(s) to memory. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + * The writer remains valid for further use when this function returns. + * + * @param Buffer A mutable memory view to write to. Must be exactly GetSaveSize() bytes. + * @return An iterator for the field(s) written to the buffer. + */ + ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer); + + ZENCORE_API void Save(BinaryWriter& Writer); + + /** + * The size of buffer (in bytes) required to serialize the fields that have been written. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + */ + ZENCORE_API uint64_t GetSaveSize() const; + + /** + * Sets the name of the next field to be written. + * + * It is not valid to call this function when writing a field inside an array. + * Names must be valid UTF-8 and must be unique within an object. + */ + ZENCORE_API CbWriter& SetName(std::string_view Name); + + /** Copy the value (not the name) of an existing field. */ + inline void AddField(std::string_view Name, const CbFieldView& Value) + { + SetName(Name); + AddField(Value); + } + + ZENCORE_API void AddField(const CbFieldView& Value); + + /** Copy the value (not the name) of an existing field. Holds a reference if owned. */ + inline void AddField(std::string_view Name, const CbField& Value) + { + SetName(Name); + AddField(Value); + } + ZENCORE_API void AddField(const CbField& Value); + + /** Begin a new object. Must have a matching call to EndObject. */ + inline void BeginObject(std::string_view Name) + { + SetName(Name); + BeginObject(); + } + ZENCORE_API void BeginObject(); + /** End an object after its fields have been written. */ + ZENCORE_API void EndObject(); + + /** Copy the value (not the name) of an existing object. */ + inline void AddObject(std::string_view Name, const CbObjectView& Value) + { + SetName(Name); + AddObject(Value); + } + ZENCORE_API void AddObject(const CbObjectView& Value); + /** Copy the value (not the name) of an existing object. Holds a reference if owned. */ + inline void AddObject(std::string_view Name, const CbObject& Value) + { + SetName(Name); + AddObject(Value); + } + ZENCORE_API void AddObject(const CbObject& Value); + + /** Begin a new array. Must have a matching call to EndArray. */ + inline void BeginArray(std::string_view Name) + { + SetName(Name); + BeginArray(); + } + ZENCORE_API void BeginArray(); + /** End an array after its fields have been written. */ + ZENCORE_API void EndArray(); + + /** Copy the value (not the name) of an existing array. */ + inline void AddArray(std::string_view Name, const CbArrayView& Value) + { + SetName(Name); + AddArray(Value); + } + ZENCORE_API void AddArray(const CbArrayView& Value); + /** Copy the value (not the name) of an existing array. Holds a reference if owned. */ + inline void AddArray(std::string_view Name, const CbArray& Value) + { + SetName(Name); + AddArray(Value); + } + ZENCORE_API void AddArray(const CbArray& Value); + + /** Write a null field. */ + inline void AddNull(std::string_view Name) + { + SetName(Name); + AddNull(); + } + ZENCORE_API void AddNull(); + + /** Write a binary field by copying Size bytes from Value. */ + inline void AddBinary(std::string_view Name, const void* Value, uint64_t Size) + { + SetName(Name); + AddBinary(Value, Size); + } + ZENCORE_API void AddBinary(const void* Value, uint64_t Size); + /** Write a binary field by copying the view. */ + inline void AddBinary(std::string_view Name, MemoryView Value) + { + SetName(Name); + AddBinary(Value); + } + inline void AddBinary(MemoryView Value) { AddBinary(Value.GetData(), Value.GetSize()); } + + /** Write a binary field by copying the buffer. Holds a reference if owned. */ + inline void AddBinary(std::string_view Name, IoBuffer Value) + { + SetName(Name); + AddBinary(std::move(Value)); + } + ZENCORE_API void AddBinary(IoBuffer Value); + ZENCORE_API void AddBinary(SharedBuffer Value); + + /** Write a string field by copying the UTF-8 value. */ + inline void AddString(std::string_view Name, std::string_view Value) + { + SetName(Name); + AddString(Value); + } + ZENCORE_API void AddString(std::string_view Value); + /** Write a string field by converting the UTF-16 value to UTF-8. */ + inline void AddString(std::string_view Name, std::wstring_view Value) + { + SetName(Name); + AddString(Value); + } + ZENCORE_API void AddString(std::wstring_view Value); + + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, int32_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(int32_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, int64_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(int64_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, uint32_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(uint32_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, uint64_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(uint64_t Value); + + /** Write a float field from a 32-bit float value. */ + inline void AddFloat(std::string_view Name, float Value) + { + SetName(Name); + AddFloat(Value); + } + ZENCORE_API void AddFloat(float Value); + + /** Write a float field from a 64-bit float value. */ + inline void AddFloat(std::string_view Name, double Value) + { + SetName(Name); + AddFloat(Value); + } + ZENCORE_API void AddFloat(double Value); + + /** Write a bool field. */ + inline void AddBool(std::string_view Name, bool bValue) + { + SetName(Name); + AddBool(bValue); + } + ZENCORE_API void AddBool(bool bValue); + + /** Write a field referencing a compact binary attachment by its hash. */ + inline void AddCompactBinaryAttachment(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddCompactBinaryAttachment(Value); + } + ZENCORE_API void AddCompactBinaryAttachment(const IoHash& Value); + + /** Write a field referencing a binary attachment by its hash. */ + inline void AddBinaryAttachment(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddBinaryAttachment(Value); + } + ZENCORE_API void AddBinaryAttachment(const IoHash& Value); + + /** Write a field referencing the attachment by its hash. */ + inline void AddAttachment(std::string_view Name, const CbAttachment& Attachment) + { + SetName(Name); + AddAttachment(Attachment); + } + ZENCORE_API void AddAttachment(const CbAttachment& Attachment); + + /** Write a hash field. */ + inline void AddHash(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddHash(Value); + } + ZENCORE_API void AddHash(const IoHash& Value); + + /** Write a UUID field. */ + inline void AddUuid(std::string_view Name, const Guid& Value) + { + SetName(Name); + AddUuid(Value); + } + ZENCORE_API void AddUuid(const Guid& Value); + + /** Write an ObjectId field. */ + inline void AddObjectId(std::string_view Name, const Oid& Value) + { + SetName(Name); + AddObjectId(Value); + } + ZENCORE_API void AddObjectId(const Oid& Value); + + /** Write a date/time field with the specified count of 100ns ticks since the epoch. */ + inline void AddDateTimeTicks(std::string_view Name, int64_t Ticks) + { + SetName(Name); + AddDateTimeTicks(Ticks); + } + ZENCORE_API void AddDateTimeTicks(int64_t Ticks); + + /** Write a date/time field. */ + inline void AddDateTime(std::string_view Name, DateTime Value) + { + SetName(Name); + AddDateTime(Value); + } + ZENCORE_API void AddDateTime(DateTime Value); + + /** Write a time span field with the specified count of 100ns ticks. */ + inline void AddTimeSpanTicks(std::string_view Name, int64_t Ticks) + { + SetName(Name); + AddTimeSpanTicks(Ticks); + } + ZENCORE_API void AddTimeSpanTicks(int64_t Ticks); + + /** Write a time span field. */ + inline void AddTimeSpan(std::string_view Name, TimeSpan Value) + { + SetName(Name); + AddTimeSpan(Value); + } + ZENCORE_API void AddTimeSpan(TimeSpan Value); + + /** Private flags that are public to work with ENUM_CLASS_FLAGS. */ + enum class StateFlags : uint8_t; + +protected: + /** Reserve the specified size up front until the format is optimized. */ + ZENCORE_API explicit CbWriter(int64_t InitialSize); + +private: + friend CbWriter& operator<<(CbWriter& Writer, std::string_view NameOrValue); + + /** Begin writing a field. May be called twice for named fields. */ + void BeginField(); + + /** Finish writing a field by writing its type. */ + void EndField(CbFieldType Type); + + /** Set the field name if valid in this state, otherwise write add a string field. */ + ZENCORE_API void SetNameOrAddString(std::string_view NameOrValue); + + /** Returns a view of the name of the active field, if any, otherwise the empty view. */ + std::string_view GetActiveName() const; + + /** Remove field types after the first to make the sequence uniform. */ + void MakeFieldsUniform(int64_t FieldBeginOffset, int64_t FieldEndOffset); + + /** State of the object, array, or top-level field being written. */ + struct WriterState + { + StateFlags Flags{}; + /** The type of the fields in the sequence if uniform, otherwise None. */ + CbFieldType UniformType{}; + /** The offset of the start of the current field. */ + int64_t Offset{}; + /** The number of fields written in this state. */ + uint64_t Count{}; + }; + +private: + // This is a prototype-quality format for the writer. Using an array of bytes is inefficient, + // and will lead to many unnecessary copies and moves of the data to resize the array, insert + // object and array sizes, and remove field types for uniform objects and uniform arrays. The + // optimized format will be a list of power-of-two blocks and an optional first block that is + // provided externally, such as on the stack. That format will store the offsets that require + // object or array sizes to be inserted and field types to be removed, and will perform those + // operations only when saving to a buffer. + std::vector<uint8_t> Data; + std::vector<WriterState> States; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A writer for compact binary object, arrays, and fields that uses a fixed-size stack buffer. + * + * @see CbWriter + */ +template<uint32_t InlineBufferSize> +class FixedCbWriter : public CbWriter +{ +public: + inline FixedCbWriter() : CbWriter(InlineBufferSize) {} + + FixedCbWriter(const FixedCbWriter&) = delete; + FixedCbWriter& operator=(const FixedCbWriter&) = delete; + +private: + // Reserve the inline buffer now even though we are unable to use it. This will avoid causing + // new stack overflows when this functionality is properly implemented in the future. + uint8_t Buffer[InlineBufferSize]; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class CbObjectWriter : public CbWriter +{ +public: + CbObjectWriter() { BeginObject(); } + + ZENCORE_API CbObject Save() + { + Finalize(); + return CbWriter::Save().AsObject(); + } + + ZENCORE_API void Save(BinaryWriter& Writer) + { + Finalize(); + return CbWriter::Save(Writer); + } + + uint64_t GetSaveSize() = delete; + + void Finalize() + { + if (m_Finalized == false) + { + EndObject(); + m_Finalized = true; + } + } + + CbObjectWriter(const CbWriter&) = delete; + CbObjectWriter& operator=(const CbWriter&) = delete; + +private: + bool m_Finalized = false; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Write the field name if valid in this state, otherwise write the string value. */ +inline CbWriter& +operator<<(CbWriter& Writer, std::string_view NameOrValue) +{ + Writer.SetNameOrAddString(NameOrValue); + return Writer; +} + +/** Write the field name if valid in this state, otherwise write the string value. */ +inline CbWriter& +operator<<(CbWriter& Writer, const char* NameOrValue) +{ + return Writer << std::string_view(NameOrValue); +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbFieldView& Value) +{ + Writer.AddField(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbField& Value) +{ + Writer.AddField(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbObjectView& Value) +{ + Writer.AddObject(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbObject& Value) +{ + Writer.AddObject(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbArrayView& Value) +{ + Writer.AddArray(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbArray& Value) +{ + Writer.AddArray(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, nullptr_t) +{ + Writer.AddNull(); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, std::wstring_view Value) +{ + Writer.AddString(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const wchar_t* Value) +{ + Writer.AddString(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, int32_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, int64_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, uint32_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, uint64_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, float Value) +{ + Writer.AddFloat(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, double Value) +{ + Writer.AddFloat(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, bool Value) +{ + Writer.AddBool(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbAttachment& Attachment) +{ + Writer.AddAttachment(Attachment); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const IoHash& Value) +{ + Writer.AddHash(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const Guid& Value) +{ + Writer.AddUuid(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const Oid& Value) +{ + Writer.AddObjectId(Value); + return Writer; +} + +ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value); +ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void usonbuilder_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compactbinarypackage.h b/zencore/include/zencore/compactbinarypackage.h new file mode 100644 index 000000000..c98ab047f --- /dev/null +++ b/zencore/include/zencore/compactbinarypackage.h @@ -0,0 +1,305 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> +#include <zencore/iohash.h> + +#include <functional> +#include <span> + +namespace zen { + +class CbWriter; +class BinaryReader; +class BinaryWriter; +class IoBuffer; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An attachment is either binary or compact binary and is identified by its hash. + * + * A compact binary attachment is also a valid binary attachment and may be accessed as binary. + * + * Attachments are serialized as one or two compact binary fields with no name. A Binary field is + * written first with its content. The content hash is omitted when the content size is zero, and + * is otherwise written as a BinaryReference or CompactBinaryReference depending on the type. + */ +class CbAttachment +{ +public: + /** Construct a null attachment. */ + CbAttachment() = default; + + /** Construct a compact binary attachment. Value is cloned if not owned. */ + inline explicit CbAttachment(CbFieldIterator Value) : CbAttachment(std::move(Value), nullptr) {} + + /** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */ + inline explicit CbAttachment(CbFieldIterator Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {} + + /** Construct a binary attachment. Value is cloned if not owned. */ + inline explicit CbAttachment(SharedBuffer Value) : CbAttachment(std::move(Value), nullptr) {} + + /** Construct a binary attachment. Value is cloned if not owned. Hash must match Value. */ + inline explicit CbAttachment(SharedBuffer Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {} + + /** Reset this to a null attachment. */ + inline void Reset() { *this = CbAttachment(); } + + /** Whether the attachment has a value. */ + inline explicit operator bool() const { return !IsNull(); } + + /** Whether the attachment has a value. */ + inline bool IsNull() const { return !Buffer; } + + /** Access the attachment as binary. Defaults to a null buffer on error. */ + ZENCORE_API SharedBuffer AsBinaryView() const; + + /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ + ZENCORE_API CbFieldIterator AsCompactBinary() const; + + /** Returns whether the attachment is binary or compact binary. */ + inline bool IsBinary() const { return !Buffer.IsNull(); } + + /** Returns whether the attachment is compact binary. */ + inline bool IsCompactBinary() const { return CompactBinary.HasValue(); } + + /** Returns the hash of the attachment value. */ + inline const IoHash& GetHash() const { return Hash; } + + /** Compares attachments by their hash. Any discrepancy in type must be handled externally. */ + inline bool operator==(const CbAttachment& Attachment) const { return Hash == Attachment.Hash; } + inline bool operator!=(const CbAttachment& Attachment) const { return Hash != Attachment.Hash; } + inline bool operator<(const CbAttachment& Attachment) const { return Hash < Attachment.Hash; } + + /** + * Load the attachment from compact binary as written by Save. + * + * The attachment references the input iterator if it is owned, and otherwise clones the value. + * + * The iterator is advanced as attachment fields are consumed from it. + */ + ZENCORE_API void Load(CbFieldIterator& Fields); + + /** + * Load the attachment from compact binary as written by Save. + */ + ZENCORE_API void Load(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc); + + /** + * Load the attachment from compact binary as written by Save. + */ + ZENCORE_API void Load(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc); + + /** Save the attachment into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(CbWriter& Writer) const; + + /** Save the attachment into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(BinaryWriter& Writer) const; + +private: + ZENCORE_API CbAttachment(CbFieldIterator Value, const IoHash* Hash); + ZENCORE_API CbAttachment(SharedBuffer Value, const IoHash* Hash); + + /** An owned buffer containing the binary or compact binary data. */ + SharedBuffer Buffer; + /** A field iterator that is valid only for compact binary attachments. */ + CbFieldViewIterator CompactBinary; + /** A hash of the attachment value. */ + IoHash Hash; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A package is a compact binary object with attachments for its external references. + * + * A package is basically a Merkle tree with compact binary as its root and other non-leaf nodes, + * and either binary or compact binary as its leaf nodes. A node references its child nodes using + * BinaryHash or FieldHash fields in its compact binary representation. + * + * It is invalid for a package to include attachments that are not referenced by its object or by + * one of its referenced compact binary attachments. When attachments are added explicitly, it is + * the responsibility of the package creator to follow this requirement. Attachments that are not + * referenced may not survive a round-trip through certain storage systems. + * + * It is valid for a package to exclude referenced attachments, but then it is the responsibility + * of the package consumer to have a mechanism for resolving those references when necessary. + * + * A package is serialized as a sequence of compact binary fields with no name. The object may be + * both preceded and followed by attachments. The object itself is written as an Object field and + * followed by its hash in a CompactBinaryReference field when the object is non-empty. A package + * ends with a Null field. The canonical order of components is the object and its hash, followed + * by the attachments ordered by hash, followed by a Null field. It is valid for the a package to + * have its components serialized in any order, provided there is at most one object and the null + * field is written last. + */ +class CbPackage +{ +public: + /** + * A function that resolves a hash to a buffer containing the data matching that hash. + * + * The resolver may return a null buffer to skip resolving an attachment for the hash. + */ + using AttachmentResolver = std::function<SharedBuffer(const IoHash& Hash)>; + + /** Construct a null package. */ + CbPackage() = default; + + /** + * Construct a package from a root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + */ + inline explicit CbPackage(CbObject InObject) { SetObject(std::move(InObject)); } + + /** + * Construct a package from a root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline explicit CbPackage(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), InResolver); } + + /** + * Construct a package from a root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + */ + inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), InObjectHash); } + + /** + * Construct a package from a root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver) + { + SetObject(std::move(InObject), InObjectHash, InResolver); + } + + /** Reset this to a null package. */ + inline void Reset() { *this = CbPackage(); } + + /** Whether the package has a non-empty object or attachments. */ + inline explicit operator bool() const { return !IsNull(); } + + /** Whether the package has an empty object and no attachments. */ + inline bool IsNull() const { return !Object.CreateIterator() && Attachments.size() == 0; } + + /** Returns the compact binary object for the package. */ + inline const CbObject& GetObject() const { return Object; } + + /** Returns the has of the compact binary object for the package. */ + inline const IoHash& GetObjectHash() const { return ObjectHash; } + + /** + * Set the root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + */ + inline void SetObject(CbObject InObject) { SetObject(std::move(InObject), nullptr, nullptr); } + + /** + * Set the root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline void SetObject(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), nullptr, &InResolver); } + + /** + * Set the root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + */ + inline void SetObject(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), &InObjectHash, nullptr); } + + /** + * Set the root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline void SetObject(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver) + { + SetObject(std::move(InObject), &InObjectHash, &InResolver); + } + + /** Returns the attachments in this package. */ + inline std::span<const CbAttachment> GetAttachments() const { return Attachments; } + + /** + * Find an attachment by its hash. + * + * @return The attachment, or null if the attachment is not found. + * @note The returned pointer is only valid until the attachments on this package are modified. + */ + ZENCORE_API const CbAttachment* FindAttachment(const IoHash& Hash) const; + + /** Find an attachment if it exists in the package. */ + inline const CbAttachment* FindAttachment(const CbAttachment& Attachment) const { return FindAttachment(Attachment.GetHash()); } + + /** Add the attachment to this package. */ + inline void AddAttachment(const CbAttachment& Attachment) { AddAttachment(Attachment, nullptr); } + + /** Add the attachment to this package, along with any references that can be resolved. */ + inline void AddAttachment(const CbAttachment& Attachment, AttachmentResolver Resolver) { AddAttachment(Attachment, &Resolver); } + + /** + * Remove an attachment by hash. + * + * @return Number of attachments removed, which will be either 0 or 1. + */ + ZENCORE_API int32_t RemoveAttachment(const IoHash& Hash); + inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); } + + /** Compares packages by their object and attachment hashes. */ + ZENCORE_API bool Equals(const CbPackage& Package) const; + inline bool operator==(const CbPackage& Package) const { return Equals(Package); } + inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); } + + /** + * Load the object and attachments from compact binary as written by Save. + * + * The object and attachments reference the input iterator, if it is owned, and otherwise clones + * the object and attachments individually to make owned copies. + * + * The iterator is advanced as object and attachment fields are consumed from it. + */ + ZENCORE_API void Load(CbFieldIterator& Fields); + + ZENCORE_API void Load(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc); + + ZENCORE_API void Load(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc); + + /** Save the object and attachments into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(CbWriter& Writer) const; + + /** Save the object and attachments into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(BinaryWriter& Writer) const; + +private: + ZENCORE_API void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver); + ZENCORE_API void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver); + + void GatherAttachments(const CbFieldViewIterator& Fields, AttachmentResolver Resolver); + + /** Attachments ordered by their hash. */ + std::vector<CbAttachment> Attachments; + CbObject Object; + IoHash ObjectHash; +}; + +void usonpackage_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compactbinaryvalidation.h b/zencore/include/zencore/compactbinaryvalidation.h new file mode 100644 index 000000000..3a3f432be --- /dev/null +++ b/zencore/include/zencore/compactbinaryvalidation.h @@ -0,0 +1,192 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> +#include <zencore/enumflags.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/refcount.h> +#include <zencore/sha1.h> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +/** Flags for validating compact binary data. */ +enum class CbValidateMode : uint32_t +{ + /** Skip validation if no other validation modes are enabled. */ + None = 0, + + /** + * Validate that the value can be read and stays inside the bounds of the memory view. + * + * This is the minimum level of validation required to be able to safely read a field, array, + * or object without the risk of crashing or reading out of bounds. + */ + Default = 1 << 0, + + /** + * Validate that object fields have unique non-empty names and array fields have no names. + * + * Name validation failures typically do not inhibit reading the input, but duplicated fields + * cannot be looked up by name other than the first, and converting to other data formats can + * fail in the presence of naming issues. + */ + Names = 1 << 1, + + /** + * Validate that fields are serialized in the canonical format. + * + * Format validation failures typically do not inhibit reading the input. Values that fail in + * this mode require more memory than in the canonical format, and comparisons of such values + * for equality are not reliable. Examples of failures include uniform arrays or objects that + * were not encoded uniformly, variable-length integers that could be encoded in fewer bytes, + * or 64-bit floats that could be encoded in 32 bits without loss of precision. + */ + Format = 1 << 2, + + /** + * Validate that there is no padding after the value before the end of the memory view. + * + * Padding validation failures have no impact on the ability to read the input, but are using + * more memory than necessary. + */ + Padding = 1 << 3, + + /** + * Validate that a package or attachment has the expected fields and matches its saved hashes. + */ + Package = 1 << 4, + + /** Perform all validation described above. */ + All = Default | Names | Format | Padding | Package, +}; + +ENUM_CLASS_FLAGS(CbValidateMode); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Flags for compact binary validation errors. Multiple flags may be combined. */ +enum class CbValidateError : uint32_t +{ + /** The input had no validation errors. */ + None = 0, + + // Mode: Default + + /** The input cannot be read without reading out of bounds. */ + OutOfBounds = 1 << 0, + /** The input has a field with an unrecognized or invalid type. */ + InvalidType = 1 << 1, + + // Mode: Names + + /** An object had more than one field with the same name. */ + DuplicateName = 1 << 2, + /** An object had a field with no name. */ + MissingName = 1 << 3, + /** An array field had a name. */ + ArrayName = 1 << 4, + + // Mode: Format + + /** A name or string payload is not valid UTF-8. */ + InvalidString = 1 << 5, + /** A size or integer payload can be encoded in fewer bytes. */ + InvalidInteger = 1 << 6, + /** A float64 payload can be encoded as a float32 without loss of precision. */ + InvalidFloat = 1 << 7, + /** An object has the same type for every field but is not uniform. */ + NonUniformObject = 1 << 8, + /** An array has the same type for every field and non-empty payloads but is not uniform. */ + NonUniformArray = 1 << 9, + + // Mode: Padding + + /** A value did not use the entire memory view given for validation. */ + Padding = 1 << 10, + + // Mode: Package + + /** The package or attachment had missing fields or fields out of order. */ + InvalidPackageFormat = 1 << 11, + /** The object or an attachment did not match the hash stored for it. */ + InvalidPackageHash = 1 << 12, + /** The package contained more than one copy of the same attachment. */ + DuplicateAttachments = 1 << 13, + /** The package contained more than one object. */ + MultiplePackageObjects = 1 << 14, + /** The package contained an object with no fields. */ + NullPackageObject = 1 << 15, + /** The package contained a null attachment. */ + NullPackageAttachment = 1 << 16, +}; + +ENUM_CLASS_FLAGS(CbValidateError); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Validate the compact binary data for one field in the view as specified by the mode flags. + * + * Only one top-level field is processed from the view, and validation recurses into any array or + * object within that field. To validate multiple consecutive top-level fields, call the function + * once for each top-level field. If the given view might contain multiple top-level fields, then + * either exclude the Padding flag from the Mode or use MeasureCompactBinary to break up the view + * into its constituent fields before validating. + * + * @param View A memory view containing at least one top-level field. + * @param Mode A combination of the flags for the types of validation to perform. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType); + +/** + * Validate the compact binary data for every field in the view as specified by the mode flags. + * + * This function expects the entire view to contain fields. Any trailing region of the view which + * does not contain a valid field will produce an OutOfBounds or InvalidType error instead of the + * Padding error that would be produced by the single field validation function. + * + * @see ValidateCompactBinary + */ +ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode); + +/** + * Validate the compact binary attachment pointed to by the view as specified by the mode flags. + * + * The attachment is validated with ValidateCompactBinary by using the validation mode specified. + * Include ECbValidateMode::Package to validate the attachment format and hash. + * + * @see ValidateCompactBinary + * + * @param View A memory view containing a package. + * @param Mode A combination of the flags for the types of validation to perform. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateCompactBinaryAttachment(MemoryView View, CbValidateMode Mode); + +/** + * Validate the compact binary package pointed to by the view as specified by the mode flags. + * + * The package, and attachments, are validated with ValidateCompactBinary by using the validation + * mode specified. Include ECbValidateMode::Package to validate the package format and hashes. + * + * @see ValidateCompactBinary + * + * @param View A memory view containing a package. + * @param Mode A combination of the flags for the types of validation to perform. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void usonvalidation_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h new file mode 100644 index 000000000..759cf8444 --- /dev/null +++ b/zencore/include/zencore/compress.h @@ -0,0 +1,53 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore/zencore.h" + +namespace zen::CompressedBuffer { + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static constexpr uint64_t DefaultBlockSize = 256 * 1024; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Method used to compress the data in a compressed buffer. */ +enum class Method : uint8_t +{ + /** Header is followed by one uncompressed block. */ + None = 0, + /** Header is followed by an array of compressed block sizes then the compressed blocks. */ + LZ4 = 4, +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Header used on every compressed buffer. Always stored in big-endian format */ +struct BufferHeader +{ + static constexpr uint32_t ExpectedMagic = 0xb7756362; + + /** A magic number to identify a compressed buffer. Always 0xb7756362 */ + uint32_t Magic = ExpectedMagic; + /** A CRC-32 used to check integrity of the buffer. Uses the polynomial 0x04c11db7 */ + uint32_t Crc32 = 0; + /** The method used to compress the buffer. Affects layout of data following the header */ + Method Method = Method::None; + /** The reserved bytes must be initialized to zero */ + uint8_t Reserved[2]{}; + /** The power of two size of every uncompressed block except the last. Size is 1 << BlockSizeExponent */ + uint8_t BlockSizeExponent = 0; + /** The number of blocks that follow the header */ + uint32_t BlockCount = 0; + /** The total size of the uncompressed data */ + uint64_t TotalRawSize = 0; + /** The total size of the compressed data including the header */ + uint64_t TotalCompressedSize = 0; +}; + +static_assert(sizeof(BufferHeader) == 32, "BufferHeader is the wrong size"); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace zen::CompressedBuffer diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h new file mode 100644 index 000000000..27c831bb1 --- /dev/null +++ b/zencore/include/zencore/endian.h @@ -0,0 +1,61 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +namespace zen { + +inline uint16_t +ByteSwap(uint16_t x) +{ + return _byteswap_ushort(x); +} + +inline uint32_t +ByteSwap(uint32_t x) +{ + return _byteswap_ulong(x); +} + +inline uint64_t +ByteSwap(uint64_t x) +{ + return _byteswap_uint64(x); +} + +inline uint16_t +FromNetworkOrder(uint16_t x) +{ + return ByteSwap(x); +} + +inline uint32_t +FromNetworkOrder(uint32_t x) +{ + return ByteSwap(x); +} + +inline uint64_t +FromNetworkOrder(uint64_t x) +{ + return ByteSwap(x); +} + +inline uint16_t +FromNetworkOrder(int16_t x) +{ + return ByteSwap(uint16_t(x)); +} + +inline uint32_t +FromNetworkOrder(int32_t x) +{ + return ByteSwap(uint32_t(x)); +} + +inline uint64_t +FromNetworkOrder(int64_t x) +{ + return ByteSwap(uint64_t(x)); +} + +} // namespace zen diff --git a/zencore/include/zencore/enumflags.h b/zencore/include/zencore/enumflags.h new file mode 100644 index 000000000..ebe747bf0 --- /dev/null +++ b/zencore/include/zencore/enumflags.h @@ -0,0 +1,61 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +namespace zen { + +// Enum class helpers + +// Defines all bitwise operators for enum classes so it can be (mostly) used as a regular flags enum +#define ENUM_CLASS_FLAGS(Enum) \ + inline Enum& operator|=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \ + inline Enum& operator&=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \ + inline Enum& operator^=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator|(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator&(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator^(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \ + inline constexpr bool operator!(Enum E) { return !(__underlying_type(Enum))E; } \ + inline constexpr Enum operator~(Enum E) { return (Enum) ~(__underlying_type(Enum))E; } + +// Friends all bitwise operators for enum classes so the definition can be kept private / protected. +#define FRIEND_ENUM_CLASS_FLAGS(Enum) \ + friend Enum& operator|=(Enum& Lhs, Enum Rhs); \ + friend Enum& operator&=(Enum& Lhs, Enum Rhs); \ + friend Enum& operator^=(Enum& Lhs, Enum Rhs); \ + friend constexpr Enum operator|(Enum Lhs, Enum Rhs); \ + friend constexpr Enum operator&(Enum Lhs, Enum Rhs); \ + friend constexpr Enum operator^(Enum Lhs, Enum Rhs); \ + friend constexpr bool operator!(Enum E); \ + friend constexpr Enum operator~(Enum E); + +template<typename Enum> +constexpr bool +EnumHasAllFlags(Enum Flags, Enum Contains) +{ + return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) == ((__underlying_type(Enum))Contains); +} + +template<typename Enum> +constexpr bool +EnumHasAnyFlags(Enum Flags, Enum Contains) +{ + return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) != 0; +} + +template<typename Enum> +void +EnumAddFlags(Enum& Flags, Enum FlagsToAdd) +{ + Flags |= FlagsToAdd; +} + +template<typename Enum> +void +EnumRemoveFlags(Enum& Flags, Enum FlagsToRemove) +{ + Flags &= ~FlagsToRemove; +} + +} // namespace zen diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h new file mode 100644 index 000000000..07c4833ff --- /dev/null +++ b/zencore/include/zencore/except.h @@ -0,0 +1,60 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/string.h> +#include <zencore/windows.h> +#include <string> + +namespace zen { + +class WindowsException : public std::exception +{ +public: + WindowsException(const char* Message) + { + m_hResult = HRESULT_FROM_WIN32(GetLastError()); + m_Message = Message; + } + + WindowsException(HRESULT hRes, const char* Message) + { + m_hResult = hRes; + m_Message = Message; + } + + WindowsException(HRESULT hRes, const char* Message, const char* Detail) + { + m_hResult = hRes; + + ExtendableStringBuilder<128> msg; + msg.Append(Message); + msg.Append(" (detail: '"); + msg.Append(Detail); + msg.Append("')"); + + m_Message = msg.c_str(); + } + + virtual const char* what() const override { return m_Message.c_str(); } + +private: + std::string m_Message; + HRESULT m_hResult; +}; + +ZENCORE_API void ThrowSystemException(HRESULT hRes, const char* Message); +inline void +ThrowSystemException(const char* Message) +{ + throw WindowsException(Message); +} + +inline void +ThrowIfFailed(HRESULT hRes, const char* Message) +{ + if (FAILED(hRes)) + ThrowSystemException(hRes, Message); +} + +} // namespace zen diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h new file mode 100644 index 000000000..b20a1e7c6 --- /dev/null +++ b/zencore/include/zencore/filesystem.h @@ -0,0 +1,74 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "stream.h" +#include "zencore.h" + +#include <filesystem> +#include <functional> + +namespace zen { + +class IoBuffer; + +/** Delete directory (after deleting any contents) + */ +ZENCORE_API bool DeleteDirectories(const wchar_t* dir); +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); + +/** Ensure directory exists. + + Will also create any required parent directories + */ +ZENCORE_API bool CreateDirectories(const wchar_t* dir); +ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); + +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectory(const wchar_t* dir); +ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); + +struct FileContents +{ + std::vector<IoBuffer> Data; + std::error_code ErrorCode; +}; + +ZENCORE_API FileContents ReadFile(std::filesystem::path Path); +ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc); +ZENCORE_API bool WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount); + +struct CopyFileOptions +{ + bool EnableClone = true; + bool MustClone = false; +}; + +ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); +ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); + +ZENCORE_API std::string ToUtf8(const std::filesystem::path& Path); + +/** + * Efficient file system traversal + * + * Uses the best available mechanism for the platform in question and could take + * advantage of any file system tracking mechanisms in the future + * + */ +class FileSystemTraversal +{ +public: + struct TreeVisitor + { + virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) = 0; + + // This should return true if we should recurse into the directory + virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) = 0; + }; + + void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor); +}; + +} // namespace zen diff --git a/zencore/include/zencore/fmtutils.h b/zencore/include/zencore/fmtutils.h new file mode 100644 index 000000000..fb5a08d56 --- /dev/null +++ b/zencore/include/zencore/fmtutils.h @@ -0,0 +1,49 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/iohash.h> +#include <zencore/string.h> +#include <zencore/uid.h> + +#include <fmt/format.h> +#include <filesystem> +#include <string_view> + +// Custom formatting for some zencore types + +template<> +struct fmt::formatter<zen::IoHash> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const zen::IoHash& Hash, FormatContext& ctx) + { + zen::IoHash::String_t String; + Hash.ToHexString(String); + return formatter<string_view>::format({String, zen::IoHash::StringLength}, ctx); + } +}; + +template<> +struct fmt::formatter<zen::Oid> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const zen::Oid& Id, FormatContext& ctx) + { + zen::StringBuilder<32> String; + Id.ToString(String); + return formatter<string_view>::format({String.c_str(), zen::Oid::StringLength}, ctx); + } +}; + +template<> +struct fmt::formatter<std::filesystem::path> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const std::filesystem::path& Path, FormatContext& ctx) + { + zen::ExtendableStringBuilder<128> String; + WideToUtf8(Path.c_str(), String); + return formatter<string_view>::format(String.ToView(), ctx); + } +}; diff --git a/zencore/include/zencore/httpclient.h b/zencore/include/zencore/httpclient.h new file mode 100644 index 000000000..4ede6839c --- /dev/null +++ b/zencore/include/zencore/httpclient.h @@ -0,0 +1,18 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/string.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +class HttpClient +{ +}; + +} // namespace zen + +void httpclient_forcelink(); // internal diff --git a/zencore/include/zencore/httpserver.h b/zencore/include/zencore/httpserver.h new file mode 100644 index 000000000..563245264 --- /dev/null +++ b/zencore/include/zencore/httpserver.h @@ -0,0 +1,373 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/enumflags.h> +#include <zencore/refcount.h> +#include <zencore/string.h> +#include <functional> +#include <gsl/gsl-lite.hpp> +#include <list> +#include <regex> +#include <span> +#include <unordered_map> +#include "zencore.h" + +namespace zen { + +class IoBuffer; +class CbObject; +class StringBuilderBase; + +enum class HttpVerb +{ + kGet = 1 << 0, + kPut = 1 << 1, + kPost = 1 << 2, + kDelete = 1 << 3, + kHead = 1 << 4, + kCopy = 1 << 5, + kOptions = 1 << 6 +}; + +gsl_DEFINE_ENUM_BITMASK_OPERATORS(HttpVerb); + +enum class HttpResponse +{ + // 1xx - Informational + + Continue = 100, //!< Indicates that the initial part of a request has been received and has not yet been rejected by the server. + SwitchingProtocols = 101, //!< Indicates that the server understands and is willing to comply with the client's request, via the + //!< Upgrade header field, for a change in the application protocol being used on this connection. + Processing = 102, //!< Is an interim response used to inform the client that the server has accepted the complete request, but has not + //!< yet completed it. + EarlyHints = 103, //!< Indicates to the client that the server is likely to send a final response with the header fields included in + //!< the informational response. + + // 2xx - Successful + + OK = 200, //!< Indicates that the request has succeeded. + Created = 201, //!< Indicates that the request has been fulfilled and has resulted in one or more new resources being created. + Accepted = 202, //!< Indicates that the request has been accepted for processing, but the processing has not been completed. + NonAuthoritativeInformation = 203, //!< Indicates that the request was successful but the enclosed payload has been modified from that + //!< of the origin server's 200 (OK) response by a transforming proxy. + NoContent = 204, //!< Indicates that the server has successfully fulfilled the request and that there is no additional content to send + //!< in the response payload body. + ResetContent = 205, //!< Indicates that the server has fulfilled the request and desires that the user agent reset the \"document + //!< view\", which caused the request to be sent, to its original state as received from the origin server. + PartialContent = 206, //!< Indicates that the server is successfully fulfilling a range request for the target resource by transferring + //!< one or more parts of the selected representation that correspond to the satisfiable ranges found in the + //!< requests's Range header field. + MultiStatus = 207, //!< Provides status for multiple independent operations. + AlreadyReported = 208, //!< Used inside a DAV:propstat response element to avoid enumerating the internal members of multiple bindings + //!< to the same collection repeatedly. [RFC 5842] + IMUsed = 226, //!< The server has fulfilled a GET request for the resource, and the response is a representation of the result of one + //!< or more instance-manipulations applied to the current instance. + + // 3xx - Redirection + + MultipleChoices = 300, //!< Indicates that the target resource has more than one representation, each with its own more specific + //!< identifier, and information about the alternatives is being provided so that the user (or user agent) can + //!< select a preferred representation by redirecting its request to one or more of those identifiers. + MovedPermanently = 301, //!< Indicates that the target resource has been assigned a new permanent URI and any future references to this + //!< resource ought to use one of the enclosed URIs. + Found = 302, //!< Indicates that the target resource resides temporarily under a different URI. + SeeOther = 303, //!< Indicates that the server is redirecting the user agent to a different resource, as indicated by a URI in the + //!< Location header field, that is intended to provide an indirect response to the original request. + NotModified = 304, //!< Indicates that a conditional GET request has been received and would have resulted in a 200 (OK) response if it + //!< were not for the fact that the condition has evaluated to false. + UseProxy = 305, //!< \deprecated \parblock Due to security concerns regarding in-band configuration of a proxy. \endparblock + //!< The requested resource MUST be accessed through the proxy given by the Location field. + TemporaryRedirect = 307, //!< Indicates that the target resource resides temporarily under a different URI and the user agent MUST NOT + //!< change the request method if it performs an automatic redirection to that URI. + PermanentRedirect = 308, //!< The target resource has been assigned a new permanent URI and any future references to this resource + //!< ought to use one of the enclosed URIs. [...] This status code is similar to 301 Moved Permanently + //!< (Section 7.3.2 of rfc7231), except that it does not allow rewriting the request method from POST to GET. + + // 4xx - Client Error + BadRequest = 400, //!< Indicates that the server cannot or will not process the request because the received syntax is invalid, + //!< nonsensical, or exceeds some limitation on what the server is willing to process. + Unauthorized = 401, //!< Indicates that the request has not been applied because it lacks valid authentication credentials for the + //!< target resource. + PaymentRequired = 402, //!< *Reserved* + Forbidden = 403, //!< Indicates that the server understood the request but refuses to authorize it. + NotFound = 404, //!< Indicates that the origin server did not find a current representation for the target resource or is not willing + //!< to disclose that one exists. + MethodNotAllowed = 405, //!< Indicates that the method specified in the request-line is known by the origin server but not supported by + //!< the target resource. + NotAcceptable = 406, //!< Indicates that the target resource does not have a current representation that would be acceptable to the + //!< user agent, according to the proactive negotiation header fields received in the request, and the server is + //!< unwilling to supply a default representation. + ProxyAuthenticationRequired = + 407, //!< Is similar to 401 (Unauthorized), but indicates that the client needs to authenticate itself in order to use a proxy. + RequestTimeout = + 408, //!< Indicates that the server did not receive a complete request message within the time that it was prepared to wait. + Conflict = 409, //!< Indicates that the request could not be completed due to a conflict with the current state of the resource. + Gone = 410, //!< Indicates that access to the target resource is no longer available at the origin server and that this condition is + //!< likely to be permanent. + LengthRequired = 411, //!< Indicates that the server refuses to accept the request without a defined Content-Length. + PreconditionFailed = + 412, //!< Indicates that one or more preconditions given in the request header fields evaluated to false when tested on the server. + PayloadTooLarge = 413, //!< Indicates that the server is refusing to process a request because the request payload is larger than the + //!< server is willing or able to process. + URITooLong = 414, //!< Indicates that the server is refusing to service the request because the request-target is longer than the + //!< server is willing to interpret. + UnsupportedMediaType = 415, //!< Indicates that the origin server is refusing to service the request because the payload is in a format + //!< not supported by the target resource for this method. + RangeNotSatisfiable = 416, //!< Indicates that none of the ranges in the request's Range header field overlap the current extent of the + //!< selected resource or that the set of ranges requested has been rejected due to invalid ranges or an + //!< excessive request of small or overlapping ranges. + ExpectationFailed = 417, //!< Indicates that the expectation given in the request's Expect header field could not be met by at least + //!< one of the inbound servers. + ImATeapot = 418, //!< Any attempt to brew coffee with a teapot should result in the error code 418 I'm a teapot. + UnprocessableEntity = 422, //!< Means the server understands the content type of the request entity (hence a 415(Unsupported Media + //!< Type) status code is inappropriate), and the syntax of the request entity is correct (thus a 400 (Bad + //!< Request) status code is inappropriate) but was unable to process the contained instructions. + Locked = 423, //!< Means the source or destination resource of a method is locked. + FailedDependency = 424, //!< Means that the method could not be performed on the resource because the requested action depended on + //!< another action and that action failed. + UpgradeRequired = 426, //!< Indicates that the server refuses to perform the request using the current protocol but might be willing to + //!< do so after the client upgrades to a different protocol. + PreconditionRequired = 428, //!< Indicates that the origin server requires the request to be conditional. + TooManyRequests = 429, //!< Indicates that the user has sent too many requests in a given amount of time (\"rate limiting\"). + RequestHeaderFieldsTooLarge = + 431, //!< Indicates that the server is unwilling to process the request because its header fields are too large. + UnavailableForLegalReasons = + 451, //!< This status code indicates that the server is denying access to the resource in response to a legal demand. + + // 5xx - Server Error + + InternalServerError = + 500, //!< Indicates that the server encountered an unexpected condition that prevented it from fulfilling the request. + NotImplemented = 501, //!< Indicates that the server does not support the functionality required to fulfill the request. + BadGateway = 502, //!< Indicates that the server, while acting as a gateway or proxy, received an invalid response from an inbound + //!< server it accessed while attempting to fulfill the request. + ServiceUnavailable = 503, //!< Indicates that the server is currently unable to handle the request due to a temporary overload or + //!< scheduled maintenance, which will likely be alleviated after some delay. + GatewayTimeout = 504, //!< Indicates that the server, while acting as a gateway or proxy, did not receive a timely response from an + //!< upstream server it needed to access in order to complete the request. + HTTPVersionNotSupported = 505, //!< Indicates that the server does not support, or refuses to support, the protocol version that was + //!< used in the request message. + VariantAlsoNegotiates = + 506, //!< Indicates that the server has an internal configuration error: the chosen variant resource is configured to engage in + //!< transparent content negotiation itself, and is therefore not a proper end point in the negotiation process. + InsufficientStorage = 507, //!< Means the method could not be performed on the resource because the server is unable to store the + //!< representation needed to successfully complete the request. + LoopDetected = 508, //!< Indicates that the server terminated an operation because it encountered an infinite loop while processing a + //!< request with "Depth: infinity". [RFC 5842] + NotExtended = 510, //!< The policy for accessing the resource has not been met in the request. [RFC 2774] + NetworkAuthenticationRequired = 511, //!< Indicates that the client needs to authenticate to gain network access. +}; + +enum class HttpContentType +{ + kBinary, + kText, + kJSON, + kCbObject, + kCbPackage +}; + +/** HTTP Server Request + */ +class HttpServerRequest +{ +public: + HttpServerRequest(); + ~HttpServerRequest(); + + // Synchronous operations + + inline [[nodiscard]] std::string_view RelativeUri() const { return m_Uri; } // Returns URI without service prefix + inline [[nodiscard]] std::string_view QueryString() const { return m_QueryString; } + inline bool IsHandled() const { return m_IsHandled; } + + struct QueryParams + { + std::vector<std::pair<std::string_view, std::string_view>> KvPairs; + + std::string_view GetValue(std::string_view ParamName) + { + for (const auto& Kv : KvPairs) + { + const std::string_view& Key = Kv.first; + + if (Key.size() == ParamName.size()) + { + if (0 == _strnicmp(Key.data(), ParamName.data(), Key.size())) + { + return Kv.second; + } + } + } + + return std::string_view(); + } + }; + + QueryParams GetQueryParams(); + + inline HttpVerb RequestVerb() const { return m_Verb; } + + const char* HeaderAccept() const; + const char* HeaderAcceptEncoding() const; + const char* HeaderContentType() const; + const char* HeaderContentEncoding() const; + inline uint64_t HeaderContentLength() const { return m_ContentLength; } + + void SetSuppressResponseBody() { m_SuppressBody = true; } + + // Asynchronous operations + + /** Read POST/PUT payload + + This will return a null buffer if the contents are not fully available yet, and the handler should + at that point return - another completion request will be issues once the contents have been received + fully. + */ + virtual IoBuffer ReadPayload() = 0; + + /** Respond with payload + + Note that this is destructive in the sense that the IoBuffer instances referred to by Blobs will be + moved into our response handler array where they are kept alive, in order to reduce ref-counting storms + */ + virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) = 0; + virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, IoBuffer Blob); + virtual void WriteResponse(HttpResponse HttpResponseCode) = 0; + + virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) = 0; + + void WriteResponse(HttpResponse HttpResponseCode, CbObject Data); + void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::string_view ResponseString); + +protected: + bool m_IsHandled = false; + bool m_SuppressBody = false; + HttpVerb m_Verb = HttpVerb::kGet; + uint64_t m_ContentLength = ~0ull; + ExtendableStringBuilder<256> m_Uri; + ExtendableStringBuilder<256> m_QueryString; +}; + +class HttpServerException : public std::exception +{ +public: + HttpServerException(const char* Message, uint32_t Error) : std::exception(Message), m_ErrorCode(Error) {} + +private: + uint32_t m_ErrorCode; +}; + +class HttpService +{ +public: + HttpService() = default; + virtual ~HttpService() = default; + + virtual const char* BaseUri() const = 0; + virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) = 0; + + // Internals + + inline void SetUriPrefixLength(size_t PrefixLength) { m_UriPrefixLength = (int)PrefixLength; } + inline int UriPrefixLength() const { return m_UriPrefixLength; } + +private: + int m_UriPrefixLength = 0; +}; + +/** HTTP server + */ +class HttpServer +{ +public: + HttpServer(); + ~HttpServer(); + + void AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler); + void AddEndpoint(HttpService& Service); + + void Initialize(int BasePort); + void Run(bool TestMode); + void RequestExit(); + +private: + struct Impl; + + RefPtr<Impl> m_Impl; +}; + +////////////////////////////////////////////////////////////////////////// + +class HttpRouterRequest +{ +public: + HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {} + + ZENCORE_API std::string GetCapture(int Index) const; + inline HttpServerRequest& ServerRequest() { return m_HttpRequest; } + +private: + using MatchResults_t = std::match_results<std::string_view::const_iterator>; + + HttpServerRequest& m_HttpRequest; + MatchResults_t m_Match; + + friend class HttpRequestRouter; +}; + +inline std::string +HttpRouterRequest::GetCapture(int Index) const +{ + ZEN_ASSERT(Index < m_Match.size()); + + return m_Match[Index]; +} + +////////////////////////////////////////////////////////////////////////// + +class HttpRequestRouter +{ +public: + typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t; + + void AddPattern(const char* Id, const char* Regex); + void RegisterRoute(const char* Regex, HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs); + bool HandleRequest(zen::HttpServerRequest& Request); + +private: + struct HandlerEntry + { + HandlerEntry(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern) + : RegEx(Regex, std::regex::icase | std::regex::ECMAScript) + , Verbs(SupportedVerbs) + , Handler(std::move(Handler)) + , Pattern(Pattern) + { + } + + ~HandlerEntry() = default; + + std::regex RegEx; + HttpVerb Verbs; + HandlerFunc_t Handler; + const char* Pattern; + }; + + std::list<HandlerEntry> m_Handlers; + std::unordered_map<std::string, std::string> m_PatternMap; +}; + +////////////////////////////////////////////////////////////////////////// +// +// HTTP Client +// + +class HttpClient +{ +}; + +} // namespace zen + +void http_forcelink(); // internal diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h new file mode 100644 index 000000000..9d39b8226 --- /dev/null +++ b/zencore/include/zencore/intmath.h @@ -0,0 +1,140 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <stdint.h> +#include <concepts> + +////////////////////////////////////////////////////////////////////////// + +#pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanReverse64) + +namespace zen { + +inline constexpr bool +IsPow2(uint64_t n) +{ + return 0 == (n & (n - 1)); +} + +/// Round an integer up to the closest integer multiplier of 'base' ('base' must be a power of two) +template<std::integral T> +T +RoundUp(T Value, auto Base) +{ + ZEN_ASSERT_SLOW(IsPow2(Base)); + return ((Value + T(Base - 1)) & (~T(Base - 1))); +} + +bool +IsMultipleOf(std::integral auto Value, auto MultiplierPow2) +{ + ZEN_ASSERT_SLOW(IsPow2(MultiplierPow2)); + return (Value & (MultiplierPow2 - 1)) == 0; +} + +inline uint64_t +NextPow2(uint64_t n) +{ + // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + + --n; + + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n |= n >> 32; + + return n + 1; +} + +static inline uint32_t +FloorLog2(uint32_t Value) +{ + // Use BSR to return the log2 of the integer + unsigned long Log2; + if (_BitScanReverse(&Log2, Value) != 0) + { + return Log2; + } + + return 0; +} + +static inline uint32_t +CountLeadingZeros(uint32_t Value) +{ + unsigned long Log2; + _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1); + return 32 - Log2; +} + +static inline uint64_t +FloorLog2_64(uint64_t Value) +{ + unsigned long Log2; + long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); + return Log2 & Mask; +} + +static inline uint64_t +CountLeadingZeros64(uint64_t Value) +{ + unsigned long Log2; + long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); + return ((63 - Log2) & Mask) | (64 & ~Mask); +} + +static inline uint64_t +CeilLogTwo64(uint64_t Arg) +{ + int64_t Bitmask = ((int64_t)(CountLeadingZeros64(Arg) << 57)) >> 63; + return (64 - CountLeadingZeros64(Arg - 1)) & (~Bitmask); +} + +static inline uint64_t +CountTrailingZeros64(uint64_t Value) +{ + if (Value == 0) + { + return 64; + } + unsigned long BitIndex; // 0-based, where the LSB is 0 and MSB is 31 + _BitScanForward64(&BitIndex, Value); // Scans from LSB to MSB + return BitIndex; +} + +////////////////////////////////////////////////////////////////////////// + +static inline bool +IsPointerAligned(const void* Ptr, uint64_t Alignment) +{ + ZEN_ASSERT_SLOW(IsPow2(Alignment)); + + return 0 == (reinterpret_cast<uintptr_t>(Ptr) & (Alignment - 1)); +} + +////////////////////////////////////////////////////////////////////////// + +#ifdef min +# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead" +#endif + +auto +Min(auto x, auto y) +{ + return x < y ? x : y; +} + +auto +Max(auto x, auto y) +{ + return x > y ? x : y; +} + +} // namespace zen diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h new file mode 100644 index 000000000..c93d27959 --- /dev/null +++ b/zencore/include/zencore/iobuffer.h @@ -0,0 +1,272 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <memory.h> +#include "refcount.h" +#include "zencore.h" + +namespace zen { + +struct IoBufferExtendedCore; + +struct IoBufferFileReference +{ + void* FileHandle; + uint64_t FileChunkOffset; + uint64_t FileChunkSize; +}; + +struct IoBufferCore +{ +public: + IoBufferCore() = default; + inline IoBufferCore(const void* DataPtr, size_t SizeBytes) : m_DataPtr(DataPtr), m_DataBytes(SizeBytes) {} + inline IoBufferCore(const IoBufferCore* Outer, const void* DataPtr, size_t SizeBytes) + : m_DataPtr(DataPtr) + , m_DataBytes(SizeBytes) + , m_OuterCore(Outer) + { + } + + ZENCORE_API explicit IoBufferCore(size_t SizeBytes); + ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment); + ZENCORE_API ~IoBufferCore(); + + // Reference counting + + inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); } + inline uint32_t Release() const + { + const uint32_t NewRefCount = AtomicDecrement(const_cast<IoBufferCore*>(this)->m_RefCount); + if (NewRefCount == 0) + { + DeleteThis(); + } + return NewRefCount; + } + + // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it + + IoBufferCore(const IoBufferCore&) = delete; + IoBufferCore(IoBufferCore&&) = delete; + IoBufferCore& operator=(const IoBufferCore&) = delete; + IoBufferCore& operator=(IoBufferCore&&) = delete; + + // + + ZENCORE_API void Materialize() const; + ZENCORE_API void DeleteThis() const; + ZENCORE_API void MakeOwned(bool Immutable = true); + + inline void EnsureDataValid() const + { + if ((m_Flags & kIsExtended) && !(m_Flags & kIsMaterialized)) + Materialize(); + } + + inline bool IsOwned() const { return !!(m_Flags & kIsOwned); } + inline bool IsImmutable() const { return !(m_Flags & kIsMutable); } + inline bool IsNull() const { return m_DataBytes == 0; } + + inline IoBufferExtendedCore* ExtendedCore(); + inline const IoBufferExtendedCore* ExtendedCore() const; + + inline const void* DataPointer() const + { + EnsureDataValid(); + return m_DataPtr; + } + + inline size_t DataBytes() const { return m_DataBytes; } + + inline void Set(const void* Ptr, size_t Sz) + { + m_DataPtr = Ptr; + m_DataBytes = Sz; + } + + inline void SetIsOwned(bool NewState) + { + if (NewState) + { + m_Flags |= kIsOwned; + } + else + { + m_Flags &= ~kIsOwned; + } + } + + inline void SetIsImmutable(bool NewState) + { + if (!NewState) + { + m_Flags |= kIsMutable; + } + else + { + m_Flags &= ~kIsMutable; + } + } + + inline uint32_t GetRefCount() const { return m_RefCount; } + +protected: + uint32_t m_RefCount = 0; + mutable uint32_t m_Flags = 0; + mutable const void* m_DataPtr = nullptr; + size_t m_DataBytes = 0; + RefPtr<const IoBufferCore> m_OuterCore; + + enum Flags + { + kIsOwned = 1 << 0, + kIsMutable = 1 << 1, + kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore + kIsMaterialized = 1 << 3, // Data pointers are valid + kLowLevelAlloc = 1 << 4, // Using direct memory allocation + }; + + void* AllocateBuffer(size_t InSize, size_t Alignment); + void FreeBuffer(); +}; + +/** + * An "Extended" core references a segment of a file + */ + +struct IoBufferExtendedCore : public IoBufferCore +{ + IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership); + IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size); + ~IoBufferExtendedCore(); + + enum ExtendedFlags + { + kOwnsFile = 1 << 8, + kOwnsMmap = 1 << 9 + }; + + void Materialize() const; + bool GetFileReference(IoBufferFileReference& OutRef) const; + +private: + void* m_FileHandle = nullptr; + uint64_t m_FileOffset = 0; + mutable void* m_MmapHandle = nullptr; + mutable void* m_MappedPointer = nullptr; +}; + +inline IoBufferExtendedCore* +IoBufferCore::ExtendedCore() +{ + if (m_Flags & kIsExtended) + { + return static_cast<IoBufferExtendedCore*>(this); + } + + return nullptr; +} + +inline const IoBufferExtendedCore* +IoBufferCore::ExtendedCore() const +{ + if (m_Flags & kIsExtended) + { + return static_cast<const IoBufferExtendedCore*>(this); + } + + return nullptr; +} + +/** + * I/O buffer + * + * This represents a reference to a payload in memory or on disk + * + */ +class IoBuffer +{ +public: + enum EAssumeOwnershipTag + { + AssumeOwnership + }; + enum ECloneTag + { + Clone + }; + enum EWrapTag + { + Wrap + }; + enum EFileTag + { + File + }; + enum EBorrowedFileTag + { + BorrowedFile + }; + + inline IoBuffer() = default; + inline IoBuffer(IoBuffer&& Rhs) noexcept = default; + inline IoBuffer(const IoBuffer& Rhs) = default; + inline IoBuffer& operator=(const IoBuffer& Rhs) = default; + inline IoBuffer& operator=(IoBuffer&& Rhs) noexcept = default; + + /** Create an uninitialized buffer of the given size + */ + ZENCORE_API explicit IoBuffer(size_t InSize); + + /** Create an uninitialized buffer of the given size with the specified alignment + */ + ZENCORE_API explicit IoBuffer(size_t InSize, uint64_t InAlignment); + + /** Create a buffer which references a sequence of bytes inside another buffer + */ + ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes); + + /** Create a buffer which references a range of bytes which we assume will live + * for the entire life time. + */ + inline IoBuffer(EWrapTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(DataPtr, SizeBytes)) {} + + inline IoBuffer(ECloneTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(SizeBytes)) + { + memcpy(const_cast<void*>(m_Core->DataPointer()), DataPtr, SizeBytes); + } + + inline IoBuffer(EAssumeOwnershipTag, const void* DataPtr, size_t Sz) : m_Core(new IoBufferCore(DataPtr, Sz)) + { + m_Core->SetIsOwned(true); + } + + ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); + ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); + + inline operator bool() const { return !m_Core->IsNull(); } + ZENCORE_API void MakeOwned() { return m_Core->MakeOwned(); } + inline bool IsOwned() const { return m_Core->IsOwned(); } + const void* Data() const { return m_Core->DataPointer(); } + const size_t Size() const { return m_Core->DataBytes(); } + ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const; + +private: + RefPtr<IoBufferCore> m_Core = new IoBufferCore; +}; + +class IoBufferBuilder +{ +public: + ZENCORE_API static IoBuffer MakeFromFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); + inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); } + +private: +}; + +void iobuffer_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/iohash.h b/zencore/include/zencore/iohash.h new file mode 100644 index 000000000..4eac7e328 --- /dev/null +++ b/zencore/include/zencore/iohash.h @@ -0,0 +1,95 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/blake3.h> +#include <zencore/memory.h> + +#include <string_view> + +namespace zen { + +class StringBuilderBase; + +/** + * Hash used for content addressable storage + * + * This is basically a BLAKE3-160 hash (note: this is probably not an officially + * recognized identifier). It is generated by computing a 32-byte BLAKE3 hash and + * picking the first 20 bytes of the resulting hash. + * + */ +struct IoHash +{ + uint8_t Hash[20]; + + static IoHash MakeFrom(const void* data /* 20 bytes */) + { + IoHash Io; + memcpy(Io.Hash, data, sizeof Io); + return Io; + } + + static IoHash HashMemory(const void* data, size_t byteCount); + static IoHash HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); } + static IoHash FromHexString(const char* string); + static IoHash FromHexString(const std::string_view string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + std::string ToHexString() const; + + static const int StringLength = 40; + typedef char String_t[StringLength + 1]; + + static IoHash Zero; // Initialized to all zeros + + inline auto operator<=>(const IoHash& rhs) const = default; + + struct Hasher + { + size_t operator()(const IoHash& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct IoHashStream +{ + /// Begin streaming hash compute (not needed on freshly constructed instance) + void Reset() { m_Blake3Stream.Reset(); } + + /// Append another chunk + IoHashStream& Append(const void* data, size_t byteCount) + { + m_Blake3Stream.Append(data, byteCount); + return *this; + } + + /// Append another chunk + IoHashStream& Append(MemoryView Data) + { + m_Blake3Stream.Append(Data.GetData(), Data.GetSize()); + return *this; + } + + /// Obtain final hash. If you wish to reuse the instance call reset() + IoHash GetHash() + { + BLAKE3 b3 = m_Blake3Stream.GetHash(); + + IoHash Io; + memcpy(Io.Hash, b3.Hash, sizeof Io.Hash); + + return Io; + } + +private: + BLAKE3Stream m_Blake3Stream; +}; + +} // namespace zen diff --git a/zencore/include/zencore/md5.h b/zencore/include/zencore/md5.h new file mode 100644 index 000000000..4ed4f6c56 --- /dev/null +++ b/zencore/include/zencore/md5.h @@ -0,0 +1,50 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <stdint.h> +#include <compare> +#include "zencore.h" + +namespace zen { + +class StringBuilderBase; + +struct MD5 +{ + uint8_t Hash[16]; + + inline auto operator<=>(const MD5& rhs) const = default; + + static const int StringLength = 40; + typedef char String_t[StringLength + 1]; + + static MD5 HashMemory(const void* data, size_t byteCount); + static MD5 FromHexString(const char* string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static MD5 Zero; // Initialized to all zeroes +}; + +/** + * Utility class for computing SHA1 hashes + */ +class MD5Stream +{ +public: + MD5Stream(); + + /// Begin streaming MD5 compute (not needed on freshly constructed MD5Stream instance) + void Reset(); + /// Append another chunk + MD5Stream& Append(const void* data, size_t byteCount); + /// Obtain final MD5 hash. If you wish to reuse the MD5Stream instance call reset() + MD5 GetHash(); + +private: +}; + +void md5_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h new file mode 100644 index 000000000..8a16126ef --- /dev/null +++ b/zencore/include/zencore/memory.h @@ -0,0 +1,213 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/thread.h> + +#include <algorithm> +#include <vector> + +namespace zen { + +class MemoryArena +{ +public: + ZENCORE_API MemoryArena(); + ZENCORE_API ~MemoryArena(); + + ZENCORE_API void* Alloc(size_t size, size_t alignment); + ZENCORE_API void Free(void* ptr); + +private: +}; + +class Memory +{ +public: + ZENCORE_API static void* Alloc(size_t size, size_t alignment = sizeof(void*)); + ZENCORE_API static void Free(void* ptr); +}; + +/** Allocator which claims fixed-size blocks from the underlying allocator. + + There is no way to free individual memory blocks. + + \note This is not thread-safe, you will need to provide synchronization yourself +*/ + +class ChunkingLinearAllocator +{ +public: + ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment = sizeof(std::max_align_t)); + ~ChunkingLinearAllocator(); + + ZENCORE_API void Reset(); + + ZENCORE_API void* Alloc(size_t Size, size_t Alignment = sizeof(void*)); + inline void Free(void* Ptr) { ZEN_UNUSED(Ptr); /* no-op */ } + +private: + uint8_t* m_ChunkCursor = nullptr; + uint64_t m_ChunkBytesRemain = 0; + const uint64_t m_ChunkSize = 0; + const uint64_t m_ChunkAlignment = 0; + std::vector<void*> m_ChunkList; +}; + +////////////////////////////////////////////////////////////////////////// + +struct MutableMemoryView +{ + MutableMemoryView() = default; + + MutableMemoryView(void* DataPtr, size_t DataSize) + : m_Data(reinterpret_cast<uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<uint8_t*>(DataPtr) + DataSize) + { + } + + inline bool IsEmpty() const { return m_Data == m_DataEnd; } + void* GetData() const { return m_Data; } + void* GetDataEnd() const { return m_DataEnd; } + size_t GetSize() const { return reinterpret_cast<uint8_t*>(m_DataEnd) - reinterpret_cast<uint8_t*>(m_Data); } + + inline bool EqualBytes(const MutableMemoryView& InView) const + { + const size_t Size = GetSize(); + + return Size == InView.GetSize() && (memcmp(m_Data, InView.m_Data, Size) == 0); + } + + /** Modifies the view by chopping the given number of bytes from the left. */ + inline void RightChopInline(uint64_t InSize) + { + const uint64_t Offset = std::min(GetSize(), InSize); + m_Data = GetDataAtOffsetNoCheck(Offset); + } + + /** Returns the left-most part of the view by taking the given number of bytes from the left. */ + constexpr inline MutableMemoryView Left(uint64_t InSize) const + { + MutableMemoryView View(*this); + View.LeftInline(InSize); + return View; + } + + /** Modifies the view to be the given number of bytes from the left. */ + constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = std::min(m_DataEnd, m_Data + InSize); } + +private: + uint8_t* m_Data = nullptr; + uint8_t* m_DataEnd = nullptr; + + /** Returns the data pointer advanced by an offset in bytes. */ + inline uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; } +}; + +struct MemoryView +{ + MemoryView() = default; + + MemoryView(const MutableMemoryView& MutableView) + : m_Data(reinterpret_cast<const uint8_t*>(MutableView.GetData())) + , m_DataEnd(m_Data + MutableView.GetSize()) + { + } + + MemoryView(const void* DataPtr, size_t DataSize) + : m_Data(reinterpret_cast<const uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<const uint8_t*>(DataPtr) + DataSize) + { + } + + MemoryView(const void* DataPtr, const void* DataEndPtr) + : m_Data(reinterpret_cast<const uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<const uint8_t*>(DataEndPtr)) + { + } + + inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); } + inline bool IsEmpty() const { return m_Data == m_DataEnd; } + const void* GetData() const { return m_Data; } + const void* GetDataEnd() const { return m_DataEnd; } + size_t GetSize() const { return reinterpret_cast<const uint8_t*>(m_DataEnd) - reinterpret_cast<const uint8_t*>(m_Data); } + inline bool operator==(const MemoryView& Rhs) const { return m_Data == Rhs.m_Data && m_DataEnd == Rhs.m_DataEnd; } + + inline bool EqualBytes(const MemoryView& InView) const + { + const size_t Size = GetSize(); + + return Size == InView.GetSize() && (memcmp(m_Data, InView.GetData(), Size) == 0); + } + + inline MemoryView& operator+=(size_t InSize) + { + RightChopInline(InSize); + return *this; + } + + /** Modifies the view by chopping the given number of bytes from the left. */ + inline void RightChopInline(uint64_t InSize) + { + const uint64_t Offset = std::min(GetSize(), InSize); + m_Data = GetDataAtOffsetNoCheck(Offset); + } + + inline MemoryView RightChop(uint64_t InSize) + { + MemoryView View(*this); + View.RightChopInline(InSize); + return View; + } + + /** Returns the left-most part of the view by taking the given number of bytes from the left. */ + constexpr inline MemoryView Left(uint64_t InSize) const + { + MemoryView View(*this); + View.LeftInline(InSize); + return View; + } + + /** Modifies the view to be the given number of bytes from the left. */ + constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = std::min(m_DataEnd, m_Data + InSize); } + + constexpr void Reset() + { + m_Data = nullptr; + m_DataEnd = nullptr; + } + +private: + const uint8_t* m_Data = nullptr; + const uint8_t* m_DataEnd = nullptr; + + /** Returns the data pointer advanced by an offset in bytes. */ + inline const uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; } +}; + +/** + * Make a non-owning view of the memory of the initializer list. + * + * This overload is only available when the element type does not need to be deduced. + */ +template<typename T> +[[nodiscard]] inline MemoryView +MakeMemoryView(std::initializer_list<typename std::type_identity<T>::type> List) +{ + return MemoryView(List.begin(), List.size() * sizeof(T)); +} + +/** Make a non-owning view of the memory of the contiguous container. */ +template<std::ranges::contiguous_range R> +[[nodiscard]] constexpr inline MemoryView +MakeMemoryView(const R& Container) +{ + const auto& Front = *std::begin(Container); + return MemoryView(std::addressof(Front), std::size(Container) * sizeof(Front)); +} + +void memory_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/meta.h b/zencore/include/zencore/meta.h new file mode 100644 index 000000000..82eb5cc30 --- /dev/null +++ b/zencore/include/zencore/meta.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +/* This file contains utility functions for meta programming + * + * Since you're in here you're probably quite observant, and you'll + * note that it's quite barren here. This is because template + * metaprogramming is awful and I try not to engage in it. However, + * sometimes these things are forced upon us. + * + */ + +namespace zen { + +/** + * Uses implicit conversion to create an instance of a specific type. + * Useful to make things clearer or circumvent unintended type deduction in templates. + * Safer than C casts and static_casts, e.g. does not allow down-casts + * + * @param Obj The object (usually pointer or reference) to convert. + * + * @return The object converted to the specified type. + */ +template<typename T> +inline T +ImplicitConv(typename std::type_identity<T>::type Obj) +{ + return Obj; +} + +} // namespace zen diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h new file mode 100644 index 000000000..de7d315f4 --- /dev/null +++ b/zencore/include/zencore/refcount.h @@ -0,0 +1,144 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +#pragma once + +#include "atomic.h" +#include "zencore.h" + +namespace zen { + +/** + * Helper base class for reference counted objects using intrusive reference counts + * + * This class is pretty straightforward but does one thing which may be unexpected: + * + * - Instances on the stack are initialized with a reference count of one to ensure + * nobody tries to accidentally delete it. (TODO: is this really useful?) + */ +class RefCounted +{ +public: + RefCounted() : m_RefCount(IsPointerToStack(this)){}; + virtual ~RefCounted() = default; + + inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); } + inline uint32_t Release() const + { + uint32_t refCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount); + if (refCount == 0) + { + delete this; + } + return refCount; + } + + // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it + + RefCounted(const RefCounted&) = delete; + RefCounted(RefCounted&&) = delete; + RefCounted& operator=(const RefCounted&) = delete; + RefCounted& operator=(RefCounted&&) = delete; + +protected: + inline uint32_t RefCount() const { return m_RefCount; } + +private: + uint32_t m_RefCount = 0; +}; + +/** + * Smart pointer for classes derived from RefCounted + */ + +template<class T> +class RefPtr +{ +public: + inline RefPtr() = default; + inline RefPtr(const RefPtr& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } + inline RefPtr(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } + inline ~RefPtr() { m_Ref && m_Ref->Release(); } + + inline explicit operator bool() const { return m_Ref != nullptr; } + inline operator T*() const { return m_Ref; } + inline T* operator->() const { return m_Ref; } + + inline std::strong_ordering operator<=>(const RefPtr& Rhs) const = default; + + inline RefPtr& operator=(T* Rhs) + { + Rhs && Rhs->AddRef(); + m_Ref && m_Ref->Release(); + m_Ref = Rhs; + return *this; + } + inline RefPtr& operator=(const RefPtr& Rhs) + { + m_Ref && m_Ref->Release(); + auto Ref = m_Ref = Rhs.m_Ref; + Ref && Ref->AddRef(); + return *this; + } + inline RefPtr& operator=(RefPtr&& Rhs) noexcept + { + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + Rhs.m_Ref = nullptr; + return *this; + } + inline RefPtr(RefPtr&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } + +private: + T* m_Ref = nullptr; +}; + +/** + * Smart pointer for classes derived from RefCounted + * + * This variant does not decay to a raw pointer + * + */ + +template<class T> +class Ref +{ +public: + inline Ref() = default; + inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } + inline Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } + inline ~Ref() { m_Ref && m_Ref->Release(); } + + inline explicit operator bool() const { return m_Ref != nullptr; } + inline T* operator->() const { return m_Ref; } + + inline std::strong_ordering operator<=>(const Ref& Rhs) const = default; + + inline Ref& operator=(T* Rhs) + { + Rhs && Rhs->AddRef(); + m_Ref && m_Ref->Release(); + m_Ref = Rhs; + return *this; + } + inline Ref& operator=(const Ref& Rhs) + { + m_Ref && m_Ref->Release(); + auto Ref = m_Ref = Rhs.m_Ref; + Ref && Ref->AddRef(); + return *this; + } + inline Ref& operator=(Ref&& Rhs) noexcept + { + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + Rhs.m_Ref = nullptr; + return *this; + } + inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } + +private: + T* m_Ref = nullptr; +}; + +void refcount_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/scopeguard.h b/zencore/include/zencore/scopeguard.h new file mode 100644 index 000000000..ba8cd3094 --- /dev/null +++ b/zencore/include/zencore/scopeguard.h @@ -0,0 +1,33 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <type_traits> +#include "zencore.h" + +namespace zen { + +template<typename T> +class ScopeGuardImpl +{ +public: + inline ScopeGuardImpl(T&& func) : m_guardFunc(func) {} + ~ScopeGuardImpl() + { + if (!m_dismissed) + m_guardFunc(); + } + + void Dismiss() { m_dismissed = true; } + +private: + bool m_dismissed = false; + T m_guardFunc; +}; + +template<typename T> +ScopeGuardImpl<T> +MakeGuard(T&& fn) +{ + return ScopeGuardImpl<T>(std::move(fn)); +} + +} // namespace zen diff --git a/zencore/include/zencore/sha1.h b/zencore/include/zencore/sha1.h new file mode 100644 index 000000000..fc26f442b --- /dev/null +++ b/zencore/include/zencore/sha1.h @@ -0,0 +1,76 @@ +// ////////////////////////////////////////////////////////// +// sha1.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +#include <stdint.h> +#include <compare> +#include "zencore.h" + +namespace zen { + +class StringBuilderBase; + +struct SHA1 +{ + uint8_t Hash[20]; + + inline auto operator<=>(const SHA1& rhs) const = default; + + static const int StringLength = 40; + typedef char String_t[StringLength + 1]; + + static SHA1 HashMemory(const void* data, size_t byteCount); + static SHA1 FromHexString(const char* string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static SHA1 Zero; // Initialized to all zeroes +}; + +/** + * Utility class for computing SHA1 hashes + */ +class SHA1Stream +{ +public: + SHA1Stream(); + + /** compute SHA1 of a memory block + + \note SHA1 class contains a slightly more convenient helper function for this use case + \see SHA1::fromMemory() + */ + SHA1 Compute(const void* data, size_t byteCount); + + /// Begin streaming SHA1 compute (not needed on freshly constructed SHA1Stream instance) + void Reset(); + /// Append another chunk + SHA1Stream& Append(const void* data, size_t byteCount); + /// Obtain final SHA1 hash. If you wish to reuse the SHA1Stream instance call reset() + SHA1 GetHash(); + +private: + void ProcessBlock(const void* data); + void ProcessBuffer(); + + enum + { + /// split into 64 byte blocks (=> 512 bits) + BlockSize = 512 / 8, + HashBytes = 20, + HashValues = HashBytes / 4 + }; + + uint64_t m_NumBytes; // size of processed data in bytes + size_t m_BufferSize; // valid bytes in m_buffer + uint8_t m_Buffer[BlockSize]; // bytes not processed yet + uint32_t m_Hash[HashValues]; +}; + +void sha1_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h new file mode 100644 index 000000000..c6206f780 --- /dev/null +++ b/zencore/include/zencore/sharedbuffer.h @@ -0,0 +1,169 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/memory.h> +#include <zencore/refcount.h> + +#include <memory.h> + +namespace zen { + +class BufferOwner : public RefCounted +{ +protected: + inline BufferOwner(void* DataPtr, uint64_t DataSize, bool IsOwned, BufferOwner* OuterBuffer = nullptr) + : m_IsOwned(IsOwned) + , m_Data(DataPtr) + , m_Size(DataSize) + , m_Outer(OuterBuffer) + { + } + + virtual ~BufferOwner(); + + // Ownership is a transitive property, and m_IsOwned currently only flags that this instance is responsible + // for managing the allocated memory, so we need to make recursive calls. Could be optimized slightly by + // adding a dedicated flag + inline bool IsOwned() const + { + if (m_IsOwned) + { + return true; + } + else + { + return m_Outer && m_Outer->IsOwned(); + } + } + + BufferOwner(const BufferOwner&) = delete; + BufferOwner& operator=(const BufferOwner&) = delete; + +private: + bool m_IsOwned; + void* m_Data; + uint64_t m_Size; + RefPtr<BufferOwner> m_Outer; + + friend class UniqueBuffer; + friend class SharedBuffer; +}; + +/** + * Reference to a memory buffer with a single owner (see std::unique_ptr) + */ +class UniqueBuffer +{ +public: + UniqueBuffer(const UniqueBuffer&) = delete; + UniqueBuffer& operator=(const UniqueBuffer&) = delete; + + UniqueBuffer() = default; + ZENCORE_API explicit UniqueBuffer(BufferOwner* Owner); + + void* GetData() { return m_buffer->m_Data; } + const void* GetData() const { return m_buffer->m_Data; } + size_t GetSize() const { return m_buffer->m_Size; } + operator MutableMemoryView() { return GetView(); } + operator MemoryView() const { return MemoryView(m_buffer->m_Data, m_buffer->m_Size); } + + MutableMemoryView GetView() { return MutableMemoryView(m_buffer->m_Data, m_buffer->m_Size); } + + /** Make an uninitialized owned buffer of the specified size. */ + ZENCORE_API static UniqueBuffer Alloc(uint64_t Size); + + /** Make a non-owned view of the input. */ + ZENCORE_API static UniqueBuffer MakeView(void* DataPtr, uint64_t Size); + +private: + RefPtr<BufferOwner> m_buffer; + + friend class SharedBuffer; +}; + +/** + * Reference to a memory buffer with shared ownership + */ +class SharedBuffer +{ +public: + SharedBuffer() = default; + ZENCORE_API explicit SharedBuffer(UniqueBuffer&&); + inline explicit SharedBuffer(BufferOwner* Owner) : m_buffer(Owner) {} + + void* GetData() + { + if (m_buffer) + { + return m_buffer->m_Data; + } + return nullptr; + } + + const void* GetData() const + { + if (m_buffer) + { + return m_buffer->m_Data; + } + return nullptr; + } + + size_t GetSize() const + { + if (m_buffer) + { + return m_buffer->m_Size; + } + return 0; + } + + ZENCORE_API void MakeOwned(); + bool IsOwned() const { return m_buffer && m_buffer->IsOwned(); } + inline explicit operator bool() const { return m_buffer; } + inline bool IsNull() const { return !m_buffer; } + inline void Reset() { m_buffer = nullptr; } + + MemoryView GetView() const + { + if (m_buffer) + { + return MemoryView(m_buffer->m_Data, m_buffer->m_Size); + } + else + { + return MemoryView(); + } + } + + operator MemoryView() const { return GetView(); } + + SharedBuffer& operator=(UniqueBuffer&& Rhs) + { + m_buffer = std::move(Rhs.m_buffer); + return *this; + } + + std::strong_ordering operator<=>(const SharedBuffer& Rhs) const = default; + + /** Make a non-owned view of the input */ + inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); } + /** Make a non-owned view of the input */ + ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size); + /** Make a non-owned view of the input */ + ZENCORE_API static SharedBuffer MakeView(MemoryView View, SharedBuffer Buffer); + /** Make am owned clone of the buffer */ + ZENCORE_API SharedBuffer Clone(); + /** Make an owned clone of the memory in the input view */ + ZENCORE_API static SharedBuffer Clone(MemoryView View); + +private: + RefPtr<BufferOwner> m_buffer; +}; + +void sharedbuffer_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/snapshot_manifest.h b/zencore/include/zencore/snapshot_manifest.h new file mode 100644 index 000000000..95e64773a --- /dev/null +++ b/zencore/include/zencore/snapshot_manifest.h @@ -0,0 +1,57 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/iohash.h> +#include <zencore/zencore.h> + +#include <filesystem> +#include <functional> +#include <string> +#include <vector> + +namespace zen { + +struct LeafNode +{ + uint64_t FileSize = 0; + uint64_t FileModifiedTime = 0; + zen::IoHash ChunkHash = zen::IoHash::Zero; + std::wstring Name; +}; + +struct TreeNode +{ + std::vector<TreeNode> Children; + std::vector<LeafNode> Leaves; + std::wstring Name; + zen::BLAKE3 ChunkHash = zen::BLAKE3::Zero; + + ZENCORE_API void VisitModifyFiles(std::function<void(LeafNode& node)> func); + ZENCORE_API void VisitFiles(std::function<void(const LeafNode& node)> func); + ZENCORE_API void Finalize(); +}; + +struct SnapshotManifest +{ + std::string Id; + TreeNode Root; + zen::BLAKE3 ChunkHash = zen::BLAKE3::Zero; + + ZENCORE_API void finalize(); +}; + +class InStream; +class OutStream; + +ZENCORE_API void ReadManifest(SnapshotManifest& Manifest, InStream& FromStream); +ZENCORE_API void WriteManifest(const SnapshotManifest& Manifest, OutStream& ToStream); +ZENCORE_API void PrintManifest(const SnapshotManifest& Manifest, OutStream& ToStream); + +// Translate a user-provided manifest specification into a file path. +// Supports hashtag syntax to implicitly refer to user documents zenfs folder +ZENCORE_API std::filesystem::path ManifestSpecToPath(const char* ManifestSpec); + +void snapshotmanifest_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h new file mode 100644 index 000000000..7290fd914 --- /dev/null +++ b/zencore/include/zencore/stats.h @@ -0,0 +1,66 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <atomic> +#include <type_traits> +#include "zencore.h" + +namespace zen { + +template<typename T> +class Gauge +{ +public: + Gauge() : m_value{0} {} + +private: + T m_value; +}; + +class Counter +{ +public: + inline void SetValue(uint64_t Value) { m_count = Value; } + inline uint64_t Value() const { return m_count; } + + inline void Increment(int64_t AddValue) { m_count += AddValue; } + inline void Decrement(int64_t SubValue) { m_count -= SubValue; } + inline void Clear() { m_count = 0; } + +private: + std::atomic_uint64_t m_count{0}; +}; + +/// <summary> +/// Exponential Weighted Moving Average +/// </summary> +class EWMA +{ +public: + /// <summary> + /// Update EWMA with new measure + /// </summary> + /// <param name="Alpha">Smoothing factor (between 0 and 1)</param> + /// <param name="Interval">Elapsed time since last</param> + /// <param name="Count">Value</param> + /// <param name="IsInitialUpdate">Whether this is the first update or not</param> + void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate); + double Rate() const; + +private: + double m_rate = 0; +}; + +/// <summary> +/// Tracks rate of events over time (i.e requests/sec) +/// </summary> +class Meter +{ +public: +private: +}; + +extern void stats_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/stream.h b/zencore/include/zencore/stream.h new file mode 100644 index 000000000..4e8c58382 --- /dev/null +++ b/zencore/include/zencore/stream.h @@ -0,0 +1,318 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/memory.h> +#include <zencore/refcount.h> +#include <zencore/thread.h> + +#include <string_view> +#include <vector> + +namespace zen { + +/** + * Basic byte stream interface + * + * This is intended as a minimal base class offering only the absolute minimum of functionality. + * + * IMPORTANT: To better support concurrency, this abstraction offers no "file pointer". Thus + * every read or write operation needs to specify the offset from which they wish to read. + * + * Most client code will likely want to use reader/writer classes like BinaryWriter/BinaryReader + * + */ +class OutStream : public RefCounted +{ +public: + virtual void Write(const void* Data, size_t ByteCount, uint64_t Offset) = 0; + virtual void Flush() = 0; +}; + +class InStream : public RefCounted +{ +public: + virtual void Read(void* DataPtr, size_t ByteCount, uint64_t Offset) = 0; + virtual uint64_t Size() const = 0; +}; + +/** + * Stream which writes into a growing memory buffer + */ +class MemoryOutStream : public OutStream +{ +public: + MemoryOutStream() = default; + ~MemoryOutStream() = default; + + virtual void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset) override; + virtual void Flush() override; + inline const uint8_t* Data() const { return m_Buffer.data(); } + inline uint64_t Size() const { return m_Buffer.size(); } + +private: + RwLock m_Lock; + std::vector<uint8_t> m_Buffer; +}; + +inline MemoryView +MakeMemoryView(const MemoryOutStream& Stream) +{ + return MemoryView(Stream.Data(), Stream.Size()); +} + +/** + * Stream which reads from a memory buffer + */ +class MemoryInStream : public InStream +{ +public: + MemoryInStream(const void* Buffer, size_t Size); + MemoryInStream(MemoryView View) : MemoryInStream(View.GetData(), View.GetSize()) {} + ~MemoryInStream() = default; + + virtual void Read(void* DataPtr, size_t ByteCount, uint64_t ReadOffset) override; + virtual uint64_t Size() const override { return m_Buffer.size(); } + inline const uint8_t* Data() const { return m_Buffer.data(); } + +private: + RwLock m_Lock; + std::vector<uint8_t> m_Buffer; +}; + +/** + * Binary stream writer + */ + +class BinaryWriter +{ +public: + inline BinaryWriter(OutStream& Stream) : m_Stream(&Stream) {} + ~BinaryWriter() = default; + + inline void Write(const void* DataPtr, size_t ByteCount) + { + m_Stream->Write(DataPtr, ByteCount, m_Offset); + m_Offset += ByteCount; + } + + uint64_t CurrentOffset() const { return m_Offset; } + +private: + RefPtr<OutStream> m_Stream; + uint64_t m_Offset = 0; +}; + +inline BinaryWriter& +operator<<(BinaryWriter& Writer, bool Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, int8_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, int16_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, int32_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, int64_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, uint8_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, uint16_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, uint32_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, uint64_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} + +/** + * Binary stream reader + */ + +class BinaryReader +{ +public: + inline BinaryReader(InStream& Stream) : m_Stream(&Stream) {} + ~BinaryReader() = default; + + inline void Read(void* DataPtr, size_t ByteCount) + { + m_Stream->Read(DataPtr, ByteCount, m_Offset); + m_Offset += ByteCount; + } + + void Seek(uint64_t Offset) + { + ZEN_ASSERT(Offset <= m_Stream->Size()); + m_Offset = Offset; + } + + void Skip(uint64_t SkipOffset) + { + ZEN_ASSERT((m_Offset + SkipOffset) <= m_Stream->Size()); + m_Offset += SkipOffset; + } + + inline uint64_t CurrentOffset() const { return m_Offset; } + inline uint64_t AvailableBytes() const { return m_Stream->Size() - m_Offset; } + +private: + RefPtr<InStream> m_Stream; + uint64_t m_Offset = 0; +}; + +inline BinaryReader& +operator>>(BinaryReader& Reader, bool& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, int8_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, int16_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, int32_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, int64_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, uint8_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, uint16_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, uint32_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, uint64_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} + +/** + * Text stream writer + */ + +class TextWriter +{ +public: + ZENCORE_API TextWriter(OutStream& Stream); + ZENCORE_API ~TextWriter(); + + ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount); + ZENCORE_API void Writef(const char* FormatString, ...); + + inline uint64_t CurrentOffset() const { return m_CurrentOffset; } + +private: + RefPtr<OutStream> m_Stream; + uint64_t m_CurrentOffset = 0; +}; + +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const char* Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const std::string_view& Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, bool Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int8_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int16_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int32_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int64_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint8_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint16_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint32_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint64_t Value); + +class IndentTextWriter : public TextWriter +{ +public: + ZENCORE_API IndentTextWriter(OutStream& stream); + ZENCORE_API ~IndentTextWriter(); + + ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount) override; + + inline void Indent(int Amount) { m_IndentAmount += Amount; } + + struct Scope + { + Scope(IndentTextWriter& Outer, int IndentAmount = 2) : m_Outer(Outer), m_IndentAmount(IndentAmount) + { + m_Outer.Indent(IndentAmount); + } + + ~Scope() { m_Outer.Indent(-m_IndentAmount); } + + private: + IndentTextWriter& m_Outer; + int m_IndentAmount; + }; + +private: + int m_IndentAmount = 0; + int m_LineCursor = 0; + char m_LineBuffer[2048]; +}; + +void stream_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/streamutil.h b/zencore/include/zencore/streamutil.h new file mode 100644 index 000000000..190cd18eb --- /dev/null +++ b/zencore/include/zencore/streamutil.h @@ -0,0 +1,118 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <fmt/format.h> +#include <zencore/string.h> +#include <string> +#include <string_view> + +#include "blake3.h" +#include "iohash.h" +#include "sha1.h" +#include "stream.h" + +namespace zen { + +ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::string_view& value); +ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::string& value); + +ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::wstring_view& value); +ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::wstring& value); +ZENCORE_API TextWriter& operator<<(TextWriter& writer, const std::wstring_view& value); + +inline BinaryWriter& +operator<<(BinaryWriter& writer, const SHA1& value) +{ + writer.Write(value.Hash, sizeof value.Hash); + return writer; +} +inline BinaryReader& +operator>>(BinaryReader& reader, SHA1& value) +{ + reader.Read(value.Hash, sizeof value.Hash); + return reader; +} +ZENCORE_API TextWriter& operator<<(TextWriter& writer, const zen::SHA1& value); + +inline BinaryWriter& +operator<<(BinaryWriter& writer, const BLAKE3& value) +{ + writer.Write(value.Hash, sizeof value.Hash); + return writer; +} +inline BinaryReader& +operator>>(BinaryReader& reader, BLAKE3& value) +{ + reader.Read(value.Hash, sizeof value.Hash); + return reader; +} +ZENCORE_API TextWriter& operator<<(TextWriter& writer, const BLAKE3& value); + +inline BinaryWriter& +operator<<(BinaryWriter& writer, const IoHash& value) +{ + writer.Write(value.Hash, sizeof value.Hash); + return writer; +} +inline BinaryReader& +operator>>(BinaryReader& reader, IoHash& value) +{ + reader.Read(value.Hash, sizeof value.Hash); + return reader; +} +ZENCORE_API TextWriter& operator<<(TextWriter& writer, const IoHash& value); + +} // namespace zen + +////////////////////////////////////////////////////////////////////////// + +template<> +struct fmt::formatter<zen::IoHash> +{ + constexpr auto parse(format_parse_context& ctx) + { + // Parse the presentation format and store it in the formatter: + auto it = ctx.begin(), end = ctx.end(); + + // Check if reached the end of the range: + if (it != end && *it != '}') + throw format_error("invalid format"); + + // Return an iterator past the end of the parsed range: + return it; + } + + template<typename FormatContext> + auto format(const zen::IoHash& h, FormatContext& ctx) + { + zen::ExtendableStringBuilder<48> String; + h.ToHexString(String); + return format_to(ctx.out(), std::string_view(String)); + } +}; + +template<> +struct fmt::formatter<zen::BLAKE3> +{ + constexpr auto parse(format_parse_context& ctx) + { + // Parse the presentation format and store it in the formatter: + auto it = ctx.begin(), end = ctx.end(); + + // Check if reached the end of the range: + if (it != end && *it != '}') + throw format_error("invalid format"); + + // Return an iterator past the end of the parsed range: + return it; + } + + template<typename FormatContext> + auto format(const zen::BLAKE3& h, FormatContext& ctx) + { + zen::ExtendableStringBuilder<80> String; + h.ToHexString(String); + return format_to(ctx.out(), std::string_view(String)); + } +}; diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h new file mode 100644 index 000000000..d7727ca08 --- /dev/null +++ b/zencore/include/zencore/string.h @@ -0,0 +1,595 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "intmath.h" +#include "zencore.h" + +#include <stdint.h> +#include <string.h> +#include <charconv> +#include <codecvt> +#include <concepts> +#include <optional> +#include <span> +#include <string_view> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +inline bool +StringEquals(const char8_t* s1, const char* s2) +{ + return strcmp(reinterpret_cast<const char*>(s1), s2) == 0; +} + +inline bool +StringEquals(const char* s1, const char* s2) +{ + return strcmp(s1, s2) == 0; +} + +inline size_t +StringLength(const char* str) +{ + return strlen(str); +} + +inline bool +StringEquals(const wchar_t* s1, const wchar_t* s2) +{ + return wcscmp(s1, s2) == 0; +} + +inline size_t +StringLength(const wchar_t* str) +{ + return wcslen(str); +} + +////////////////////////////////////////////////////////////////////////// +// File name helpers +// + +ZENCORE_API const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr); + +////////////////////////////////////////////////////////////////////////// +// Text formatting of numbers +// + +ZENCORE_API bool ToString(std::span<char> Buffer, uint64_t Num); +ZENCORE_API bool ToString(std::span<char> Buffer, int64_t Num); + +struct TextNumBase +{ + inline const char* c_str() const { return m_Buffer; } + inline operator std::string_view() const { return std::string_view(m_Buffer); } + +protected: + char m_Buffer[24]; +}; + +struct IntNum : public TextNumBase +{ + inline IntNum(std::unsigned_integral auto Number) { ToString(m_Buffer, uint64_t(Number)); } + inline IntNum(std::signed_integral auto Number) { ToString(m_Buffer, int64_t(Number)); } +}; + +////////////////////////////////////////////////////////////////////////// +// +// Quick-and-dirty string builder. Good enough for me, but contains traps +// and not-quite-ideal behaviour especially when mixing character types etc +// + +template<typename C> +class StringBuilderImpl +{ +public: + StringBuilderImpl() = default; + ZENCORE_API ~StringBuilderImpl(); + + StringBuilderImpl(const StringBuilderImpl&) = delete; + StringBuilderImpl(const StringBuilderImpl&&) = delete; + const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete; + const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete; + + StringBuilderImpl& Append(C OneChar) + { + EnsureCapacity(1); + + *m_CurPos++ = OneChar; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const std::string_view& String) + { + const size_t len = String.size(); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = String[i]; + + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const std::u8string_view& String) + { + const size_t len = String.size(); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = String[i]; + + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const char* NulTerminatedString) + { + size_t StringLen = StringLength(NulTerminatedString); + + return AppendAscii({NulTerminatedString, StringLen}); + } + + inline StringBuilderImpl& Append(const char8_t* NulTerminatedString) + { + // This is super hacky and not fully functional - needs better + // solution + if constexpr (sizeof(C) == 1) + { + size_t len = StringLength((const char*)NulTerminatedString); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = C(NulTerminatedString[i]); + + m_CurPos += len; + } + else + { + ZEN_NOT_IMPLEMENTED(); + } + + return *this; + } + + inline StringBuilderImpl& AppendAsciiRange(const char* BeginString, const char* EndString) + { + EnsureCapacity(EndString - BeginString); + + while (BeginString != EndString) + *m_CurPos++ = *BeginString++; + + return *this; + } + + inline StringBuilderImpl& Append(const C* NulTerminatedString) + { + size_t Len = StringLength(NulTerminatedString); + + EnsureCapacity(Len); + memcpy(m_CurPos, NulTerminatedString, Len * sizeof(C)); + m_CurPos += Len; + + return *this; + } + + inline StringBuilderImpl& Append(const C* NulTerminatedString, size_t MaxChars) + { + size_t len = Min(MaxChars, StringLength(NulTerminatedString)); + + EnsureCapacity(len); + memcpy(m_CurPos, NulTerminatedString, len * sizeof(C)); + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendRange(const C* BeginString, const C* EndString) + { + size_t Len = EndString - BeginString; + + EnsureCapacity(Len); + memcpy(m_CurPos, BeginString, Len * sizeof(C)); + m_CurPos += Len; + + return *this; + } + + inline StringBuilderImpl& Append(const std::basic_string_view<C>& String) + { + return AppendRange(String.data(), String.data() + String.size()); + } + + inline const C* c_str() const + { + EnsureNulTerminated(); + return m_Base; + } + + inline C* Data() + { + EnsureNulTerminated(); + return m_Base; + } + + inline const C* Data() const + { + EnsureNulTerminated(); + return m_Base; + } + + inline size_t Size() const { return m_CurPos - m_Base; } + inline bool IsDynamic() const { return m_IsDynamic; } + inline void Reset() { m_CurPos = m_Base; } + + inline StringBuilderImpl& operator<<(uint64_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int64_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint32_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int32_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint16_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int16_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint8_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int8_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + + inline StringBuilderImpl& operator<<(const char* str) { return AppendAscii(str); } + inline StringBuilderImpl& operator<<(const std::string_view str) { return AppendAscii(str); } + inline StringBuilderImpl& operator<<(const std::u8string_view str) { return AppendAscii(str); } + inline StringBuilderImpl& operator<<(bool v) + { + using namespace std::literals; + if (v) + { + return AppendAscii("true"sv); + } + return AppendAscii("false"sv); + } + +protected: + inline void Init(C* Base, size_t Capacity) + { + m_Base = m_CurPos = Base; + m_End = Base + Capacity; + } + + inline void EnsureNulTerminated() const { *m_CurPos = '\0'; } + + inline void EnsureCapacity(size_t ExtraRequired) + { + // precondition: we know the current buffer has enough capacity + // for the existing string including NUL terminator + + if ((m_CurPos + ExtraRequired) < m_End) + return; + + Extend(ExtraRequired); + } + + ZENCORE_API void Extend(size_t ExtraCapacity); + ZENCORE_API void* AllocBuffer(size_t ByteCount); + ZENCORE_API void FreeBuffer(void* Buffer, size_t ByteCount); + + ZENCORE_API [[noreturn]] void Fail(const char* FailReason); // note: throws exception + + C* m_Base; + C* m_CurPos; + C* m_End; + bool m_IsDynamic = false; + bool m_IsExtendable = false; +}; + +////////////////////////////////////////////////////////////////////////// + +extern template StringBuilderImpl<char>; + +class StringBuilderBase : public StringBuilderImpl<char> +{ +public: + inline StringBuilderBase(char* bufferPointer, size_t bufferCapacity) { Init(bufferPointer, bufferCapacity); } + inline ~StringBuilderBase() = default; + + // Note that we don't need a terminator for the string_view so we avoid calling data() here + inline operator std::string_view() const { return std::string_view(m_Base, m_CurPos - m_Base); } + inline std::string_view ToView() const { return std::string_view(m_Base, m_CurPos - m_Base); } + inline std::string ToString() const { return std::string{Data(), Size()}; } + + inline void AppendCodepoint(uint32_t cp) + { + if (cp < 0x80) // one octet + { + Append(static_cast<char8_t>(cp)); + } + else if (cp < 0x800) + { + EnsureCapacity(2); // two octets + m_CurPos[0] = static_cast<char8_t>((cp >> 6) | 0xc0); + m_CurPos[1] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 2; + } + else if (cp < 0x10000) + { + EnsureCapacity(3); // three octets + m_CurPos[0] = static_cast<char8_t>((cp >> 12) | 0xe0); + m_CurPos[1] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80); + m_CurPos[2] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 3; + } + else + { + EnsureCapacity(4); // four octets + m_CurPos[0] = static_cast<char8_t>((cp >> 18) | 0xf0); + m_CurPos[1] = static_cast<char8_t>(((cp >> 12) & 0x3f) | 0x80); + m_CurPos[2] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80); + m_CurPos[3] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 4; + } + } +}; + +template<size_t N> +class StringBuilder : public StringBuilderBase +{ +public: + inline StringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) {} + inline ~StringBuilder() = default; + +private: + char m_StringBuffer[N]; +}; + +template<size_t N> +class ExtendableStringBuilder : public StringBuilderBase +{ +public: + inline ExtendableStringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) { m_IsExtendable = true; } + inline ~ExtendableStringBuilder() = default; + +private: + char m_StringBuffer[N]; +}; + +////////////////////////////////////////////////////////////////////////// + +extern template StringBuilderImpl<wchar_t>; + +class WideStringBuilderBase : public StringBuilderImpl<wchar_t> +{ +public: + inline WideStringBuilderBase(wchar_t* BufferPointer, size_t BufferCapacity) { Init(BufferPointer, BufferCapacity); } + inline ~WideStringBuilderBase() = default; + + inline operator std::wstring_view() const { return std::wstring_view{Data(), Size()}; } + inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; } + inline std::wstring toString() const { return std::wstring{Data(), Size()}; } + + inline StringBuilderImpl& operator<<(const std::u16string_view str) { return Append((const wchar_t*)str.data(), str.size()); } + inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); } + using StringBuilderImpl:: operator<<; +}; + +template<size_t N> +class WideStringBuilder : public WideStringBuilderBase +{ +public: + inline WideStringBuilder() : WideStringBuilderBase(m_Buffer, N) {} + ~WideStringBuilder() = default; + +private: + wchar_t m_Buffer[N]; +}; + +template<size_t N> +class ExtendableWideStringBuilder : public WideStringBuilderBase +{ +public: + inline ExtendableWideStringBuilder() : WideStringBuilderBase(m_Buffer, N) { m_IsExtendable = true; } + ~ExtendableWideStringBuilder() = default; + +private: + wchar_t m_Buffer[N]; +}; + +////////////////////////////////////////////////////////////////////////// + +void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out); +void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out); +void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out); +std::wstring Utf8ToWide(const std::string_view& wstr); + +void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out); +std::string WideToUtf8(const wchar_t* wstr); +void WideToUtf8(const std::u16string_view& wstr, StringBuilderBase& out); +void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out); +std::string WideToUtf8(const std::wstring_view Wstr); + +/// <summary> +/// Parse hex string into a byte buffer +/// </summary> +/// <param name="string">Input string</param> +/// <param name="characterCount">Number of characters in string</param> +/// <param name="outPtr">Pointer to output buffer</param> +/// <returns>true if the input consisted of all valid hexadecimal characters</returns> + +inline bool +ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr) +{ + ZEN_ASSERT((CharacterCount & 1) == 0); + + auto char2nibble = [](char c) { + uint8_t c8 = uint8_t(c - '0'); + + if (c8 < 10) + return c8; + + c8 -= 'A' - '0' - 10; + + if (c8 < 16) + return c8; + + c8 -= 'a' - 'A'; + + if (c8 < 16) + return c8; + + return uint8_t(0xff); + }; + + uint8_t allBits = 0; + + while (CharacterCount) + { + uint8_t n0 = char2nibble(InputString[0]); + uint8_t n1 = char2nibble(InputString[1]); + + allBits |= n0 | n1; + + *OutPtr = (n0 << 4) | n1; + + OutPtr += 1; + InputString += 2; + CharacterCount -= 2; + } + + return (allBits & 0x80) == 0; +} + +inline void +ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString) +{ + const char hexchars[] = "0123456789abcdef"; + + while (ByteCount--) + { + uint8_t byte = *InputData++; + + *OutString++ = hexchars[byte >> 4]; + *OutString++ = hexchars[byte & 15]; + } +} + +////////////////////////////////////////////////////////////////////////// +// Format numbers for humans +// + +ZENCORE_API size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer); +ZENCORE_API size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer); +ZENCORE_API size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer); +ZENCORE_API size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer); +ZENCORE_API size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer); + +struct NiceBase +{ + inline const char* c_str() const { return m_Buffer; } + inline operator std::string_view() const { return std::string_view(m_Buffer); } + +protected: + char m_Buffer[16]; +}; + +struct NiceNum : public NiceBase +{ + inline NiceNum(uint64_t Num) { NiceNumToBuffer(Num, m_Buffer); } +}; + +struct NiceBytes : public NiceBase +{ + inline NiceBytes(uint64_t Num) { NiceBytesToBuffer(Num, m_Buffer); } +}; + +struct NiceByteRate : public NiceBase +{ + inline NiceByteRate(uint64_t Bytes, uint64_t TimeMilliseconds) { NiceByteRateToBuffer(Bytes, TimeMilliseconds, m_Buffer); } +}; + +struct NiceLatencyNs : public NiceBase +{ + inline NiceLatencyNs(uint64_t Milliseconds) { NiceLatencyNsToBuffer(Milliseconds, m_Buffer); } +}; + +struct NiceTimeSpanMs : public NiceBase +{ + inline NiceTimeSpanMs(uint64_t Milliseconds) { NiceTimeSpanMsToBuffer(Milliseconds, m_Buffer); } +}; + +////////////////////////////////////////////////////////////////////////// + +inline std::string +NiceRate(uint64_t Num, uint32_t DurationMilliseconds, const char* Unit = "B") +{ + char Buffer[32]; + + if (DurationMilliseconds) + { + NiceNumToBuffer(Num * 1000 / DurationMilliseconds, Buffer); + } + else + { + strcpy_s(Buffer, "0"); + } + + strcat_s(Buffer, Unit); + strcat_s(Buffer, "/s"); + + return Buffer; +} + +////////////////////////////////////////////////////////////////////////// + +template<std::integral T> +std::optional<T> +ParseInt(const std::string_view& Input) +{ + T Out; + const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out); + if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range) + { + return std::nullopt; + } + return Out; +} + +////////////////////////////////////////////////////////////////////////// + +void string_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h new file mode 100644 index 000000000..48afad33f --- /dev/null +++ b/zencore/include/zencore/thread.h @@ -0,0 +1,118 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +namespace zen { + +/** + * Reader-writer lock + * + * - A single thread may hold an exclusive lock at any given moment + * + * - Multiple threads may hold shared locks, but only if no thread has + * acquired an exclusive lock + */ +class RwLock +{ +public: + ZENCORE_API void AcquireShared(); + ZENCORE_API void ReleaseShared(); + + ZENCORE_API void AcquireExclusive(); + ZENCORE_API void ReleaseExclusive(); + + struct SharedLockScope + { + SharedLockScope(RwLock& lock) : m_Lock(lock) { m_Lock.AcquireShared(); } + ~SharedLockScope() { m_Lock.ReleaseShared(); } + + private: + RwLock& m_Lock; + }; + + struct ExclusiveLockScope + { + ExclusiveLockScope(RwLock& lock) : m_Lock(lock) { m_Lock.AcquireExclusive(); } + ~ExclusiveLockScope() { m_Lock.ReleaseExclusive(); } + + private: + RwLock& m_Lock; + }; + +private: + void* m_Srw = nullptr; +}; + +/** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore') + */ +class Event +{ +public: + ZENCORE_API Event(); + ZENCORE_API ~Event(); + + Event(Event&& Rhs) : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; } + + Event(const Event& Rhs) = delete; + Event& operator=(const Event& Rhs) = delete; + + inline Event& operator=(Event&& Rhs) + { + m_EventHandle = Rhs.m_EventHandle; + Rhs.m_EventHandle = nullptr; + return *this; + } + + ZENCORE_API void Set(); + ZENCORE_API void Reset(); + ZENCORE_API bool Wait(int TimeoutMs = -1); + +protected: + explicit Event(void* EventHandle) : m_EventHandle(EventHandle) {} + + void* m_EventHandle = nullptr; +}; + +/** Basic abstraction of an IPC mechanism (aka 'binary semaphore') + */ +class NamedEvent : public Event +{ +public: + ZENCORE_API explicit NamedEvent(std::string_view EventName); + ZENCORE_API explicit NamedEvent(std::u8string_view EventName); +}; + +/** Basic process abstraction + */ +class Process +{ +public: + ZENCORE_API Process(); + + Process(const Process&) = delete; + Process& operator=(const Process&) = delete; + + ZENCORE_API ~Process(); + + ZENCORE_API void Initialize(int Pid); + ZENCORE_API void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle + ZENCORE_API bool IsRunning() const; + ZENCORE_API bool IsValid() const; + ZENCORE_API bool Wait(int TimeoutMs = -1); + ZENCORE_API void Terminate(int ExitCode); + inline int Pid() const { return m_Pid; } + +private: + void* m_ProcessHandle = nullptr; + int m_Pid = 0; +}; + +ZENCORE_API bool IsProcessRunning(int pid); + +ZENCORE_API void Sleep(int ms); + +void thread_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/timer.h b/zencore/include/zencore/timer.h new file mode 100644 index 000000000..c9122eb44 --- /dev/null +++ b/zencore/include/zencore/timer.h @@ -0,0 +1,41 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <intrin.h> +#include <stdint.h> +#include "zencore.h" + +namespace zen { + +// High frequency timers + +ZENCORE_API uint64_t GetHifreqTimerValue(); +ZENCORE_API uint64_t GetHifreqTimerFrequency(); +ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init + +class Stopwatch +{ +public: + Stopwatch() : m_StartValue(GetHifreqTimerValue()) {} + + inline uint64_t getElapsedTimeMs() { return (GetHifreqTimerValue() - m_StartValue) * 1000 / GetHifreqTimerFrequency(); } + + inline void reset() { m_StartValue = GetHifreqTimerValue(); } + +private: + uint64_t m_StartValue; +}; + +// CPU timers + +inline uint64_t +GetCpuTimerValue() +{ + unsigned int foo; + return __rdtscp(&foo); +} + +void timer_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/trace.h b/zencore/include/zencore/trace.h new file mode 100644 index 000000000..191ce4a3a --- /dev/null +++ b/zencore/include/zencore/trace.h @@ -0,0 +1,91 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <inttypes.h> +#include "zencore.h" + +#pragma section("trace_events", read) +#define U_TRACE_DECL __declspec(allocate("trace_events")) + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +struct TraceSite +{ + const char* sourceFile; + const uint32_t sourceLine; + const uint32_t flags; +}; + +struct TraceEvent +{ + const TraceSite* site; + ThreadId_t threadId; + const char* message; +}; + +enum TraceFlags +{ + kTrace_Debug = 1 << 0, + kTrace_Info = 1 << 1, + kTrace_Warn = 1 << 2, + kTrace_Error = 1 << 3, + kTrace_Fatal = 1 << 4, + + kTrace_Trace = 1 << 7, +}; + +class Tracer +{ +public: + void Log(const TraceEvent& e); + + __forceinline uint32_t Accept(const TraceSite& e) const { return (m_acceptFlags & e.flags); } + +private: + uint32_t m_acceptFlags = ~0u; +}; + +ZENCORE_API extern Tracer g_globalTracer; + +/** Trace event handler + */ +class TraceHandler +{ +public: + virtual void Trace(const TraceEvent& e) = 0; + +private: +}; + +ZENCORE_API static void TraceBroadcast(const TraceEvent& e); + +void trace_forcelink(); // internal + +} // namespace zen + +__forceinline zen::Tracer& +CurrentTracer() +{ + return zen::g_globalTracer; +} + +#define U_LOG_GENERIC(msg, flags) \ + do \ + { \ + zen::Tracer& t = CurrentTracer(); \ + static U_TRACE_DECL constexpr zen::TraceSite traceSite{__FILE__, __LINE__, flags}; \ + const zen::TraceEvent traceEvent = {&traceSite, 0u, msg}; \ + if (t.Accept(traceSite)) \ + t.Log(traceEvent); \ + } while (false) + +////////////////////////////////////////////////////////////////////////// + +#define U_LOG_DEBUG(msg) U_LOG_GENERIC(msg, zen::kTrace_Debug) +#define U_LOG_INFO(msg) U_LOG_GENERIC(msg, zen::kTrace_Info) +#define U_LOG_WARN(msg) U_LOG_GENERIC(msg, zen::kTrace_Warn) +#define U_LOG_ERROR(msg) U_LOG_GENERIC(msg, zen::kTrace_Error) +#define U_LOG_FATAL(msg) U_LOG_GENERIC(msg, zen::kTrace_Fatal) diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h new file mode 100644 index 000000000..a793b160a --- /dev/null +++ b/zencore/include/zencore/uid.h @@ -0,0 +1,78 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> +#include <compare> + +namespace zen { + +class StringBuilderBase; + +/** Object identifier + + Can be used as a GUID essentially, but is more compact (12 bytes) and as such + is more susceptible to collisions than a 16-byte GUID but also I don't expect + the population to be large so in practice the risk should be minimal due to + how the identifiers work. + + Similar in spirit to MongoDB ObjectId + + When serialized, object identifiers generated in a given session in sequence + will sort in chronological order since the timestamp is in the MSB in big + endian format. This makes it suitable as a database key since most indexing + structures work better when keys are inserted in lexicographically + increasing order. + + The current layout is basically: + + |----------------|----------------|----------------| + | timestamp | serial # | run id | + |----------------|----------------|----------------| + MSB LSB + + - Timestamp is a unsigned 32-bit value (seconds since Jan 1 1970) + - Serial # is another unsigned 32-bit value which is assigned a (strong) + random number at initialization time which is incremented when a new Oid + is generated + - The run id is generated from a strong random number generator + at initialization time and stays fixed for the duration of the program + + Timestamp and serial are stored in memory in such a way that they can be + ordered lexicographically. I.e they are in big-endian byte order. + + */ + +struct Oid +{ + static const int StringLength = 24; + typedef char String_t[StringLength + 1]; + + static void Initialize(); + [[nodiscard]] static Oid NewOid(); + + const Oid& Generate(); + [[nodiscard]] static Oid FromHexString(const std::string_view String); + StringBuilderBase& ToString(StringBuilderBase& OutString) const; + + auto operator<=>(const Oid& rhs) const = default; + + static const Oid Zero; // Min (can be used to signify a "null" value, or for open range queries) + static const Oid Max; // Max (can be used for open range queries) + + struct Hasher + { + size_t operator()(const Oid& id) const + { + const size_t seed = id.OidBits[0]; + return (seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1]) | (uint64_t(id.OidBits[2]) << 32); + } + }; + + // You should not assume anything about these words + uint32_t OidBits[3]; +}; + +extern void uid_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/varint.h b/zencore/include/zencore/varint.h new file mode 100644 index 000000000..0c40dd66b --- /dev/null +++ b/zencore/include/zencore/varint.h @@ -0,0 +1,255 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "intmath.h" + +namespace zen { + +// Variable-Length Integer Encoding +// +// ZigZag encoding is used to convert signed integers into unsigned integers in a way that allows +// integers with a small magnitude to have a smaller encoded representation. +// +// An unsigned integer is encoded into 1-9 bytes based on its magnitude. The first byte indicates +// how many additional bytes are used by the number of leading 1-bits that it has. The additional +// bytes are stored in big endian order, and the most significant bits of the value are stored in +// the remaining bits in the first byte. The encoding of the first byte allows the reader to skip +// over the encoded integer without consuming its bytes individually. +// +// Encoded unsigned integers sort the same in a byte-wise comparison as when their decoded values +// are compared. The same property does not hold for signed integers due to ZigZag encoding. +// +// 32-bit inputs encode to 1-5 bytes. +// 64-bit inputs encode to 1-9 bytes. +// +// 0x0000'0000'0000'0000 - 0x0000'0000'0000'007f : 0b0_______ 1 byte +// 0x0000'0000'0000'0080 - 0x0000'0000'0000'3fff : 0b10______ 2 bytes +// 0x0000'0000'0000'4000 - 0x0000'0000'001f'ffff : 0b110_____ 3 bytes +// 0x0000'0000'0020'0000 - 0x0000'0000'0fff'ffff : 0b1110____ 4 bytes +// 0x0000'0000'1000'0000 - 0x0000'0007'ffff'ffff : 0b11110___ 5 bytes +// 0x0000'0008'0000'0000 - 0x0000'03ff'ffff'ffff : 0b111110__ 6 bytes +// 0x0000'0400'0000'0000 - 0x0001'ffff'ffff'ffff : 0b1111110_ 7 bytes +// 0x0002'0000'0000'0000 - 0x00ff'ffff'ffff'ffff : 0b11111110 8 bytes +// 0x0100'0000'0000'0000 - 0xffff'ffff'ffff'ffff : 0b11111111 9 bytes +// +// Encoding Examples +// -42 => ZigZag => 0x53 => 0x53 +// 42 => ZigZag => 0x54 => 0x54 +// 0x1 => 0x01 +// 0x12 => 0x12 +// 0x123 => 0x81 0x23 +// 0x1234 => 0x92 0x34 +// 0x12345 => 0xc1 0x23 0x45 +// 0x123456 => 0xd2 0x34 0x56 +// 0x1234567 => 0xe1 0x23 0x45 0x67 +// 0x12345678 => 0xf0 0x12 0x34 0x56 0x78 +// 0x123456789 => 0xf1 0x23 0x45 0x67 0x89 +// 0x123456789a => 0xf8 0x12 0x34 0x56 0x78 0x9a +// 0x123456789ab => 0xfb 0x23 0x45 0x67 0x89 0xab +// 0x123456789abc => 0xfc 0x12 0x34 0x56 0x78 0x9a 0xbc +// 0x123456789abcd => 0xfd 0x23 0x45 0x67 0x89 0xab 0xcd +// 0x123456789abcde => 0xfe 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde +// 0x123456789abcdef => 0xff 0x01 0x23 0x45 0x67 0x89 0xab 0xcd 0xef +// 0x123456789abcdef0 => 0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0 + +/** + * Measure the length in bytes (1-9) of an encoded variable-length integer. + * + * @param InData A variable-length encoding of an (signed or unsigned) integer. + * @return The number of bytes used to encode the integer, in the range 1-9. + */ +inline uint32_t +MeasureVarUInt(const void* InData) +{ + return CountLeadingZeros(uint8_t(~*static_cast<const uint8_t*>(InData))) - 23; +} + +/** Measure the length in bytes (1-9) of an encoded variable-length integer. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(const void* InData) +{ + return MeasureVarUInt(InData); +} + +/** Measure the number of bytes (1-5) required to encode the 32-bit input. */ +inline uint32_t +MeasureVarUInt(uint32_t InValue) +{ + return uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1); +} + +/** Measure the number of bytes (1-9) required to encode the 64-bit input. */ +inline uint32_t +MeasureVarUInt(uint64_t InValue) +{ + return uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9)); +} + +/** Measure the number of bytes (1-5) required to encode the 32-bit input. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(int32_t InValue) +{ + return MeasureVarUInt(uint32_t((InValue >> 31) ^ (InValue << 1))); +} + +/** Measure the number of bytes (1-9) required to encode the 64-bit input. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(int64_t InValue) +{ + return MeasureVarUInt(uint64_t((InValue >> 63) ^ (InValue << 1))); +} + +/** + * Read a variable-length unsigned integer. + * + * @param InData A variable-length encoding of an unsigned integer. + * @param OutByteCount The number of bytes consumed from the input. + * @return An unsigned integer. + */ +inline uint64_t +ReadVarUInt(const void* InData, uint32_t& OutByteCount) +{ + const uint32_t ByteCount = MeasureVarUInt(InData); + OutByteCount = ByteCount; + + const uint8_t* InBytes = static_cast<const uint8_t*>(InData); + uint64_t Value = *InBytes++ & uint8_t(0xff >> ByteCount); + switch (ByteCount - 1) + { + case 8: + Value <<= 8; + Value |= *InBytes++; + case 7: + Value <<= 8; + Value |= *InBytes++; + case 6: + Value <<= 8; + Value |= *InBytes++; + case 5: + Value <<= 8; + Value |= *InBytes++; + case 4: + Value <<= 8; + Value |= *InBytes++; + case 3: + Value <<= 8; + Value |= *InBytes++; + case 2: + Value <<= 8; + Value |= *InBytes++; + case 1: + Value <<= 8; + Value |= *InBytes++; + default: + return Value; + } +} + +/** + * Read a variable-length signed integer. + * + * @param InData A variable-length encoding of a signed integer. + * @param OutByteCount The number of bytes consumed from the input. + * @return A signed integer. + */ +inline int64_t +ReadVarInt(const void* InData, uint32_t& OutByteCount) +{ + const uint64_t Value = ReadVarUInt(InData, OutByteCount); + return -int64_t(Value & 1) ^ int64_t(Value >> 1); +} + +/** + * Write a variable-length unsigned integer. + * + * @param InValue An unsigned integer to encode. + * @param OutData A buffer of at least 5 bytes to write the output to. + * @return The number of bytes used in the output. + */ +inline uint32_t +WriteVarUInt(uint32_t InValue, void* OutData) +{ + const uint32_t ByteCount = MeasureVarUInt(InValue); + uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1; + switch (ByteCount - 1) + { + case 4: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 3: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 2: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 1: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + default: + break; + } + *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue); + return ByteCount; +} + +/** + * Write a variable-length unsigned integer. + * + * @param InValue An unsigned integer to encode. + * @param OutData A buffer of at least 9 bytes to write the output to. + * @return The number of bytes used in the output. + */ +inline uint32_t +WriteVarUInt(uint64_t InValue, void* OutData) +{ + const uint32_t ByteCount = MeasureVarUInt(InValue); + uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1; + switch (ByteCount - 1) + { + case 8: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 7: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 6: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 5: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 4: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 3: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 2: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 1: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + default: + break; + } + *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue); + return ByteCount; +} + +/** Write a variable-length signed integer. \see \ref WriteVarUInt */ +inline uint32_t +WriteVarInt(int32_t InValue, void* OutData) +{ + const uint32_t Value = uint32_t((InValue >> 31) ^ (InValue << 1)); + return WriteVarUInt(Value, OutData); +} + +/** Write a variable-length signed integer. \see \ref WriteVarUInt */ +inline uint32_t +WriteVarInt(int64_t InValue, void* OutData) +{ + const uint64_t Value = uint64_t((InValue >> 63) ^ (InValue << 1)); + return WriteVarUInt(Value, OutData); +} + +} // namespace zen diff --git a/zencore/include/zencore/windows.h b/zencore/include/zencore/windows.h new file mode 100644 index 000000000..8888bf757 --- /dev/null +++ b/zencore/include/zencore/windows.h @@ -0,0 +1,10 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- +#ifndef NOMINMAX +# define NOMINMAX // We don't want your min/max macros +#endif +#define WIN32_LEAN_AND_MEAN +#include <windows.h> diff --git a/zencore/include/zencore/xxhash.h b/zencore/include/zencore/xxhash.h new file mode 100644 index 000000000..5407755df --- /dev/null +++ b/zencore/include/zencore/xxhash.h @@ -0,0 +1,87 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/memory.h> + +#include <xxh3.h> +#include <string_view> + +namespace zen { + +class StringBuilderBase; + +/** + * XXH3 hash + */ +struct XXH3_128 +{ + uint8_t Hash[16]; + + static XXH3_128 MakeFrom(const void* data /* 16 bytes */) + { + XXH3_128 Xx; + memcpy(Xx.Hash, data, sizeof Xx); + return Xx; + } + + static inline XXH3_128 HashMemory(const void* data, size_t byteCount) + { + XXH3_128 Hash; + XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits(data, byteCount)); + return Hash; + } + static XXH3_128 HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); } + static XXH3_128 FromHexString(const char* string); + static XXH3_128 FromHexString(const std::string_view string); + const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static const int StringLength = 32; + typedef char String_t[StringLength + 1]; + + static XXH3_128 Zero; // Initialized to all zeros + + inline auto operator<=>(const XXH3_128& rhs) const = default; + + struct Hasher + { + size_t operator()(const XXH3_128& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct XXH3_128Stream +{ + /// Begin streaming hash compute (not needed on freshly constructed instance) + void Reset() { memset(&m_State, 0, sizeof m_State); } + + /// Append another chunk + XXH3_128Stream& Append(const void* Data, size_t ByteCount) + { + XXH3_128bits_update(&m_State, Data, ByteCount); + return *this; + } + + /// Append another chunk + XXH3_128Stream& Append(MemoryView Data) { return Append(Data.GetData(), Data.GetSize()); } + + /// Obtain final hash. If you wish to reuse the instance call reset() + XXH3_128 GetHash() + { + XXH3_128 Hash; + XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits_digest(&m_State)); + return Hash; + } + +private: + XXH3_state_s m_State{}; +}; + +} // namespace zen diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h new file mode 100644 index 000000000..4a448776b --- /dev/null +++ b/zencore/include/zencore/zencore.h @@ -0,0 +1,134 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <cinttypes> +#include <exception> +#include <string> + +////////////////////////////////////////////////////////////////////////// +// Platform +// + +#define ZEN_PLATFORM_WINDOWS 1 +#define ZEN_PLATFORM_LINUX 0 +#define ZEN_PLATFORM_MACOS 0 + +////////////////////////////////////////////////////////////////////////// +// Compiler +// + +#ifdef _MSC_VER +# define ZEN_COMPILER_MSC 1 +#endif + +#ifndef ZEN_COMPILER_MSC +# define ZEN_COMPILER_MSC 0 +#endif + +#ifndef ZEN_COMPILER_CLANG +# define ZEN_COMPILER_CLANG 0 +#endif + +////////////////////////////////////////////////////////////////////////// +// Build flavor +// + +#ifdef NDEBUG +# define ZEN_BUILD_DEBUG 0 +# define ZEN_BUILD_RELEASE 1 +#else +# define ZEN_BUILD_DEBUG 1 +# define ZEN_BUILD_RELEASE 0 +#endif + +////////////////////////////////////////////////////////////////////////// + +#define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1 + +////////////////////////////////////////////////////////////////////////// +// Assert +// + +namespace zen { + +class AssertException : public std::exception +{ +public: + AssertException(const char* Msg) : m_Msg(Msg) {} + + [[nodiscard]] virtual char const* what() const override { return m_Msg.c_str(); } + +private: + std::string m_Msg; +}; + +} // namespace zen + +#define ZEN_ASSERT(x, ...) \ + do \ + { \ + if (x) \ + break; \ + throw ::zen::AssertException{#x}; \ + } while (false) + +#ifndef NDEBUG +# define ZEN_ASSERT_SLOW(x, ...) \ + do \ + { \ + if (x) \ + break; \ + throw ::zen::AssertException{#x}; \ + } while (false) +#else +# define ZEN_ASSERT_SLOW(x, ...) +#endif + +////////////////////////////////////////////////////////////////////////// + +#ifdef __clang__ +template<typename T> +auto ZenArrayCountHelper(T& t) -> typename std::enable_if<__is_array(T), char (&)[sizeof(t) / sizeof(t[0]) + 1]>::Type; +#else +template<typename T, uint32_t N> +char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; +#endif + +#define ZEN_ARRAY_COUNT(array) (sizeof(ZenArrayCountHelper(array)) - 1) + +////////////////////////////////////////////////////////////////////////// + +#define ZEN_NOINLINE __declspec(noinline) +#define ZEN_UNUSED(...) ((void)__VA_ARGS__) +#define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false) +#define ZENCORE_API // Placeholder to allow DLL configs in the future + +ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is within the stack of the currently executing thread +ZENCORE_API bool IsApplicationExitRequested(); +ZENCORE_API void RequestApplicationExit(int ExitCode); + +ZENCORE_API void zencore_forcelinktests(); + +////////////////////////////////////////////////////////////////////////// + +#if ZEN_COMPILER_MSC +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on)) +#else +#endif + +// Set up optimization control macros, now that we have both the build settings and the platform macros +#define ZEN_DISABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL + +#if ZEN_BUILD_DEBUG +# define ZEN_ENABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL +#else +# define ZEN_ENABLE_OPTIMIZATION ZEN_ENABLE_OPTIMIZATION_ACTUAL +#endif + +#define ZEN_ENABLE_OPTIMIZATION_ALWAYS ZEN_ENABLE_OPTIMIZATION_ACTUAL + +////////////////////////////////////////////////////////////////////////// + +using ThreadId_t = uint32_t; |