diff options
| author | Stefan Boberg <[email protected]> | 2021-05-11 13:05:39 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2021-05-11 13:05:39 +0200 |
| commit | f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa (patch) | |
| tree | 1daf7621e110d48acd5e12e3073ce48ef0dd11b2 /zencore | |
| download | zen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.tar.xz zen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.zip | |
Adding zenservice code
Diffstat (limited to 'zencore')
69 files changed, 17427 insertions, 0 deletions
diff --git a/zencore/blake3.cpp b/zencore/blake3.cpp new file mode 100644 index 000000000..ec5d496d5 --- /dev/null +++ b/zencore/blake3.cpp @@ -0,0 +1,153 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/blake3.h> + +#include <zencore/string.h> +#include <zencore/zencore.h> +#include "../3rdparty/BLAKE3/c/blake3.h" + +#pragma comment(lib, "blake3.lib") + +#include <doctest/doctest.h> +#include <string.h> + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +void +blake3_forcelink() +{ +} + +BLAKE3 BLAKE3::Zero; // Initialized to all zeroes + +BLAKE3 +BLAKE3::HashMemory(const void* data, size_t byteCount) +{ + BLAKE3 b3; + + blake3_hasher b3h; + blake3_hasher_init(&b3h); + blake3_hasher_update(&b3h, data, byteCount); + blake3_hasher_finalize(&b3h, b3.Hash, sizeof b3.Hash); + + return b3; +} + +BLAKE3 +BLAKE3::FromHexString(const char* string) +{ + BLAKE3 b3; + + ParseHexBytes(string, 2 * sizeof b3.Hash, b3.Hash); + + return b3; +} + +const char* +BLAKE3::ToHexString(char* outString /* 40 characters + NUL terminator */) const +{ + ToHexBytes(Hash, sizeof(BLAKE3), outString); + outString[2 * sizeof(BLAKE3)] = '\0'; + + return outString; +} + +StringBuilderBase& +BLAKE3::ToHexString(StringBuilderBase& outBuilder) const +{ + char str[65]; + ToHexString(str); + + outBuilder.AppendRange(str, &str[65]); + + return outBuilder; +} + +BLAKE3Stream::BLAKE3Stream() +{ + blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState); + static_assert(sizeof(blake3_hasher) <= sizeof(m_HashState)); + blake3_hasher_init(b3h); +} + +void +BLAKE3Stream::Reset() +{ + blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState); + blake3_hasher_init(b3h); +} + +BLAKE3Stream& +BLAKE3Stream::Append(const void* data, size_t byteCount) +{ + blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState); + blake3_hasher_update(b3h, data, byteCount); + + return *this; +} + +BLAKE3 +BLAKE3Stream::GetHash() +{ + BLAKE3 b3; + + blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState); + blake3_hasher_finalize(b3h, b3.Hash, sizeof b3.Hash); + + return b3; +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +doctest::String +toString(const BLAKE3& value) +{ + char text[2 * sizeof(BLAKE3) + 1]; + value.ToHexString(text); + + return text; +} + +TEST_CASE("BLAKE3") +{ + SUBCASE("Basics") + { + BLAKE3 b3 = BLAKE3::HashMemory(nullptr, 0); + CHECK(BLAKE3::FromHexString("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262") == b3); + + BLAKE3::String_t b3s; + std::string b3ss = b3.ToHexString(b3s); + CHECK(b3ss == "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"); + } + + SUBCASE("hashes") + { + CHECK(BLAKE3::FromHexString("00307ced6a8b278d5e3a9f77b138d0e9d2209717c9d45b205f427a73565cc5fb") == BLAKE3::HashMemory("abc123", 6)); + CHECK(BLAKE3::FromHexString("a7142c8c3905cd11b1e35105c7ac588b75d6798822f71e1145187ad46f3e8df4") == + BLAKE3::HashMemory("1234567890123456789012345678901234567890", 40)); + CHECK(BLAKE3::FromHexString("70e708532559265c4662d0285e5e0a4be8bd972bd1f255a93ddf342243adc427") == + BLAKE3::HashMemory("The HttpSendHttpResponse function sends an HTTP response to the specified HTTP request.", 87)); + } + + SUBCASE("streamHashes") + { + auto streamHash = [](const void* data, size_t dataBytes) -> BLAKE3 { + BLAKE3Stream b3s; + b3s.Append(data, dataBytes); + return b3s.GetHash(); + }; + + CHECK(BLAKE3::FromHexString("00307ced6a8b278d5e3a9f77b138d0e9d2209717c9d45b205f427a73565cc5fb") == streamHash("abc123", 6)); + CHECK(BLAKE3::FromHexString("a7142c8c3905cd11b1e35105c7ac588b75d6798822f71e1145187ad46f3e8df4") == + streamHash("1234567890123456789012345678901234567890", 40)); + CHECK(BLAKE3::FromHexString("70e708532559265c4662d0285e5e0a4be8bd972bd1f255a93ddf342243adc427") == + streamHash("The HttpSendHttpResponse function sends an HTTP response to the specified HTTP request.", 87)); + } +} + +} // namespace zen diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp new file mode 100644 index 000000000..4ee9e9281 --- /dev/null +++ b/zencore/compactbinary.cpp @@ -0,0 +1,1279 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/compactbinary.h" + +#include <zencore/endian.h> +#include <zencore/stream.h> +#include <zencore/trace.h> + +#include <doctest/doctest.h> +#include <ryml/ryml.hpp> +#include <string_view> + +namespace zen { + +const int DaysPerMonth[] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; +const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}; + +bool +IsLeapYear(int Year) +{ + if ((Year % 4) == 0) + { + return (((Year % 100) != 0) || ((Year % 400) == 0)); + } + + return false; +} + +void +DateTime::Set(int Year, int Month, int Day, int Hour, int Minute, int Second, int MilliSecond) +{ + int TotalDays = 0; + + if ((Month > 2) && IsLeapYear(Year)) + { + ++TotalDays; + } + + --Year; // the current year is not a full year yet + --Month; // the current month is not a full month yet + + TotalDays += Year * 365; + TotalDays += Year / 4; // leap year day every four years... + TotalDays -= Year / 100; // ...except every 100 years... + TotalDays += Year / 400; // ...but also every 400 years + TotalDays += DaysToMonth[Month]; // days in this year up to last month + TotalDays += Day - 1; // days in this month minus today + + Ticks = TotalDays * TimeSpan::TicksPerDay + Hour * TimeSpan::TicksPerHour + Minute * TimeSpan::TicksPerMinute + + Second * TimeSpan::TicksPerSecond + MilliSecond * TimeSpan::TicksPerMillisecond; +} + +void +TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano) +{ + int64_t TotalTicks = 0; + + TotalTicks += Days * TicksPerDay; + TotalTicks += Hours * TicksPerHour; + TotalTicks += Minutes * TicksPerMinute; + TotalTicks += Seconds * TicksPerSecond; + TotalTicks += FractionNano / NanosecondsPerTick; + + Ticks = TotalTicks; +} + +////////////////////////////////////////////////////////////////////////// + +namespace usonprivate { + + static constexpr const uint8_t GEmptyObjectPayload[] = {uint8_t(CbFieldType::Object), 0x00}; + static constexpr const uint8_t GEmptyArrayPayload[] = {uint8_t(CbFieldType::Array), 0x01, 0x00}; + + template<typename T> + static constexpr inline T ReadUnaligned(const void* const Memory) + { +#if PLATFORM_SUPPORTS_UNALIGNED_LOADS + return *static_cast<const T*>(Memory); +#else + T Value; + memcpy(&Value, Memory, sizeof(Value)); + return Value; +#endif + } +} // namespace usonprivate + +////////////////////////////////////////////////////////////////////////// + +CbFieldView::CbFieldView(const void* DataPointer, CbFieldType FieldType) +{ + const uint8_t* Bytes = static_cast<const uint8_t*>(DataPointer); + const CbFieldType LocalType = CbFieldTypeOps::HasFieldType(FieldType) ? (CbFieldType(*Bytes++) | CbFieldType::HasFieldType) : FieldType; + + uint32_t NameLenByteCount = 0; + const uint64_t NameLen64 = CbFieldTypeOps::HasFieldName(LocalType) ? ReadVarUInt(Bytes, NameLenByteCount) : 0; + Bytes += NameLen64 + NameLenByteCount; + + Type = LocalType; + NameLen = uint32_t(std::clamp<uint64_t>(NameLen64, 0, ~uint32_t(0))); + Payload = Bytes; +} + +void +CbFieldView::IterateAttachments(std::function<void(CbFieldView)> Visitor) const +{ + switch (CbFieldTypeOps::GetType(Type)) + { + case CbFieldType::Object: + case CbFieldType::UniformObject: + return CbObjectView::FromFieldView(*this).IterateAttachments(Visitor); + case CbFieldType::Array: + case CbFieldType::UniformArray: + return CbArrayView::FromFieldView(*this).IterateAttachments(Visitor); + case CbFieldType::CompactBinaryAttachment: + case CbFieldType::BinaryAttachment: + return Visitor(*this); + default: + return; + } +} + +CbObjectView +CbFieldView::AsObjectView() +{ + if (CbFieldTypeOps::IsObject(Type)) + { + Error = CbFieldError::None; + return CbObjectView::FromFieldView(*this); + } + else + { + Error = CbFieldError::TypeError; + return CbObjectView(); + } +} + +CbArrayView +CbFieldView::AsArrayView() +{ + if (CbFieldTypeOps::IsArray(Type)) + { + Error = CbFieldError::None; + return CbArrayView::FromFieldView(*this); + } + else + { + Error = CbFieldError::TypeError; + return CbArrayView(); + } +} + +MemoryView +CbFieldView::AsBinaryView(const MemoryView Default) +{ + if (CbFieldTypeOps::IsBinary(Type)) + { + const uint8_t* const PayloadBytes = static_cast<const uint8_t*>(Payload); + uint32_t ValueSizeByteCount; + const uint64_t ValueSize = ReadVarUInt(PayloadBytes, ValueSizeByteCount); + + Error = CbFieldError::None; + return MemoryView(PayloadBytes + ValueSizeByteCount, ValueSize); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +std::string_view +CbFieldView::AsString(const std::string_view Default) +{ + if (CbFieldTypeOps::IsString(Type)) + { + const char* const PayloadChars = static_cast<const char*>(Payload); + uint32_t ValueSizeByteCount; + const uint64_t ValueSize = ReadVarUInt(PayloadChars, ValueSizeByteCount); + + if (ValueSize >= (uint64_t(1) << 31)) + { + Error = CbFieldError::RangeError; + return Default; + } + + Error = CbFieldError::None; + return std::string_view(PayloadChars + ValueSizeByteCount, ValueSize); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +uint64_t +CbFieldView::AsInteger(const uint64_t Default, const IntegerParams Params) +{ + if (CbFieldTypeOps::IsInteger(Type)) + { + // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero. + const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1); + const uint64_t IsNegative = uint8_t(Type) & 1; + + uint32_t MagnitudeByteCount; + const uint64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount); + const uint64_t Value = Magnitude ^ -int64_t(IsNegative); + + const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned); + Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError; + + const uint64_t UseValueMask = -int64_t(IsInRange); + return (Value & UseValueMask) | (Default & ~UseValueMask); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +float +CbFieldView::AsFloat(const float Default) +{ + switch (CbFieldTypeOps::GetType(Type)) + { + case CbFieldType::IntegerPositive: + case CbFieldType::IntegerNegative: + { + const uint64_t IsNegative = uint8_t(Type) & 1; + constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*FLT_MANT_DIG*/ 24) - 1); + + uint32_t MagnitudeByteCount; + const int64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount) + IsNegative; + const uint64_t IsInRange = !(Magnitude & OutOfRangeMask); + Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError; + return IsInRange ? float(IsNegative ? -Magnitude : Magnitude) : Default; + } + case CbFieldType::Float32: + { + Error = CbFieldError::None; + const uint32_t Value = FromNetworkOrder(usonprivate::ReadUnaligned<uint32_t>(Payload)); + return reinterpret_cast<const float&>(Value); + } + case CbFieldType::Float64: + Error = CbFieldError::RangeError; + return Default; + default: + Error = CbFieldError::TypeError; + return Default; + } +} + +double +CbFieldView::AsDouble(const double Default) +{ + switch (CbFieldTypeOps::GetType(Type)) + { + case CbFieldType::IntegerPositive: + case CbFieldType::IntegerNegative: + { + const uint64_t IsNegative = uint8_t(Type) & 1; + constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*DBL_MANT_DIG*/ 53) - 1); + + uint32_t MagnitudeByteCount; + const int64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount) + IsNegative; + const uint64_t IsInRange = !(Magnitude & OutOfRangeMask); + Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError; + return IsInRange ? double(IsNegative ? -Magnitude : Magnitude) : Default; + } + case CbFieldType::Float32: + { + Error = CbFieldError::None; + const uint32_t Value = FromNetworkOrder(usonprivate::ReadUnaligned<uint32_t>(Payload)); + return reinterpret_cast<const float&>(Value); + } + case CbFieldType::Float64: + { + Error = CbFieldError::None; + const uint64_t Value = FromNetworkOrder(usonprivate::ReadUnaligned<uint64_t>(Payload)); + return reinterpret_cast<const double&>(Value); + } + default: + Error = CbFieldError::TypeError; + return Default; + } +} + +bool +CbFieldView::AsBool(const bool bDefault) +{ + const CbFieldType LocalType = Type; + const bool bIsBool = CbFieldTypeOps::IsBool(LocalType); + Error = bIsBool ? CbFieldError::None : CbFieldError::TypeError; + return (uint8_t(bIsBool) & uint8_t(LocalType) & 1) | ((!bIsBool) & bDefault); +} + +IoHash +CbFieldView::AsCompactBinaryAttachment(const IoHash& Default) +{ + if (CbFieldTypeOps::IsCompactBinaryAttachment(Type)) + { + Error = CbFieldError::None; + return IoHash::MakeFrom(Payload); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +IoHash +CbFieldView::AsBinaryAttachment(const IoHash& Default) +{ + if (CbFieldTypeOps::IsBinaryAttachment(Type)) + { + Error = CbFieldError::None; + return IoHash::MakeFrom(Payload); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +IoHash +CbFieldView::AsAttachment(const IoHash& Default) +{ + if (CbFieldTypeOps::IsAttachment(Type)) + { + Error = CbFieldError::None; + return IoHash::MakeFrom(Payload); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +IoHash +CbFieldView::AsHash(const IoHash& Default) +{ + if (CbFieldTypeOps::IsHash(Type)) + { + Error = CbFieldError::None; + return IoHash::MakeFrom(Payload); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +Guid +CbFieldView::AsUuid() +{ + return AsUuid(Guid()); +} + +Guid +CbFieldView::AsUuid(const Guid& Default) +{ + if (CbFieldTypeOps::IsUuid(Type)) + { + Error = CbFieldError::None; + Guid Value; + memcpy(&Value, Payload, sizeof(Guid)); + Value.A = FromNetworkOrder(Value.A); + Value.B = FromNetworkOrder(Value.B); + Value.C = FromNetworkOrder(Value.C); + Value.D = FromNetworkOrder(Value.D); + return Value; + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +Oid +CbFieldView::AsObjectId() +{ + return AsObjectId(Oid()); +} + +Oid +CbFieldView::AsObjectId(const Oid& Default) +{ + if (CbFieldTypeOps::IsObjectId(Type)) + { + Error = CbFieldError::None; + Oid Value; + memcpy(&Value, Payload, sizeof(Oid)); + return Value; + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +int64_t +CbFieldView::AsDateTimeTicks(const int64_t Default) +{ + if (CbFieldTypeOps::IsDateTime(Type)) + { + Error = CbFieldError::None; + return FromNetworkOrder(usonprivate::ReadUnaligned<int64_t>(Payload)); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +DateTime +CbFieldView::AsDateTime() +{ + return DateTime(AsDateTimeTicks(0)); +} + +DateTime +CbFieldView::AsDateTime(DateTime Default) +{ + return DateTime(AsDateTimeTicks(Default.GetTicks())); +} + +int64_t +CbFieldView::AsTimeSpanTicks(const int64_t Default) +{ + if (CbFieldTypeOps::IsTimeSpan(Type)) + { + Error = CbFieldError::None; + return FromNetworkOrder(usonprivate::ReadUnaligned<int64_t>(Payload)); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +TimeSpan +CbFieldView::AsTimeSpan() +{ + return TimeSpan(AsTimeSpanTicks(0)); +} + +TimeSpan +CbFieldView::AsTimeSpan(TimeSpan Default) +{ + return TimeSpan(AsTimeSpanTicks(Default.GetTicks())); +} + +uint64_t +CbFieldView::GetSize() const +{ + return sizeof(CbFieldType) + GetViewNoType().GetSize(); +} + +uint64_t +CbFieldView::GetPayloadSize() const +{ + switch (CbFieldTypeOps::GetType(Type)) + { + case CbFieldType::None: + case CbFieldType::Null: + return 0; + case CbFieldType::Object: + case CbFieldType::UniformObject: + case CbFieldType::Array: + case CbFieldType::UniformArray: + case CbFieldType::Binary: + case CbFieldType::String: + { + uint32_t PayloadSizeByteCount; + const uint64_t PayloadSize = ReadVarUInt(Payload, PayloadSizeByteCount); + return PayloadSize + PayloadSizeByteCount; + } + case CbFieldType::IntegerPositive: + case CbFieldType::IntegerNegative: + return MeasureVarUInt(Payload); + case CbFieldType::Float32: + return 4; + case CbFieldType::Float64: + return 8; + case CbFieldType::BoolFalse: + case CbFieldType::BoolTrue: + return 0; + case CbFieldType::CompactBinaryAttachment: + case CbFieldType::BinaryAttachment: + case CbFieldType::Hash: + return 20; + case CbFieldType::Uuid: + return 16; + case CbFieldType::ObjectId: + return 12; + case CbFieldType::DateTime: + case CbFieldType::TimeSpan: + return 8; + default: + return 0; + } +} + +IoHash +CbFieldView::GetHash() const +{ + IoHashStream HashStream; + GetHash(HashStream); + return HashStream.GetHash(); +} + +void +CbFieldView::GetHash(IoHashStream& Hash) const +{ + const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type); + Hash.Append(&SerializedType, sizeof(SerializedType)); + auto View = GetViewNoType(); + Hash.Append(View.GetData(), View.GetSize()); +} + +bool +CbFieldView::Equals(const CbFieldView& Other) const +{ + return CbFieldTypeOps::GetSerializedType(Type) == CbFieldTypeOps::GetSerializedType(Other.Type) && + GetViewNoType().EqualBytes(Other.GetViewNoType()); +} + +void +CbFieldView::CopyTo(MutableMemoryView Buffer) const +{ + const MemoryView Source = GetViewNoType(); + ZEN_ASSERT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize()); + // TEXT("A buffer of %" UINT64_FMT " bytes was provided when %" UINT64_FMT " bytes are required"), + // Buffer.GetSize(), + // sizeof(CbFieldType) + Source.GetSize()); + *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetSerializedType(Type); + Buffer.RightChopInline(sizeof(CbFieldType)); + memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize()); +} + +void +CbFieldView::CopyTo(BinaryWriter& Ar) const +{ + const MemoryView Source = GetViewNoType(); + CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type); + Ar.Write(&SerializedType, sizeof(SerializedType)); + Ar.Write(Source.GetData(), Source.GetSize()); +} + +MemoryView +CbFieldView::GetView() const +{ + const uint32_t TypeSize = CbFieldTypeOps::HasFieldType(Type) ? sizeof(CbFieldType) : 0; + const uint32_t NameSize = CbFieldTypeOps::HasFieldName(Type) ? NameLen + MeasureVarUInt(NameLen) : 0; + const uint64_t PayloadSize = GetPayloadSize(); + return MemoryView(static_cast<const uint8_t*>(Payload) - TypeSize - NameSize, TypeSize + NameSize + PayloadSize); +} + +MemoryView +CbFieldView::GetViewNoType() const +{ + const uint32_t NameSize = CbFieldTypeOps::HasFieldName(Type) ? NameLen + MeasureVarUInt(NameLen) : 0; + const uint64_t PayloadSize = GetPayloadSize(); + return MemoryView(static_cast<const uint8_t*>(Payload) - NameSize, NameSize + PayloadSize); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +CbArrayView::CbArrayView() : CbFieldView(usonprivate::GEmptyArrayPayload) +{ +} + +uint64_t +CbArrayView::Num() const +{ + const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload()); + PayloadBytes += MeasureVarUInt(PayloadBytes); + uint32_t NumByteCount; + return ReadVarUInt(PayloadBytes, NumByteCount); +} + +CbFieldViewIterator +CbArrayView::CreateViewIterator() const +{ + const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload()); + uint32_t PayloadSizeByteCount; + const uint64_t PayloadSize = ReadVarUInt(PayloadBytes, PayloadSizeByteCount); + PayloadBytes += PayloadSizeByteCount; + const uint64_t NumByteCount = MeasureVarUInt(PayloadBytes); + if (PayloadSize > NumByteCount) + { + const void* const PayloadEnd = PayloadBytes + PayloadSize; + PayloadBytes += NumByteCount; + const CbFieldType UniformType = + CbFieldTypeOps::GetType(GetType()) == CbFieldType::UniformArray ? CbFieldType(*PayloadBytes++) : CbFieldType::HasFieldType; + return CbFieldViewIterator::MakeRange(MemoryView(PayloadBytes, PayloadEnd), UniformType); + } + return CbFieldViewIterator(); +} + +void +CbArrayView::VisitFields(ICbVisitor&) +{ +} + +uint64_t +CbArrayView::GetSize() const +{ + return sizeof(CbFieldType) + GetPayloadSize(); +} + +IoHash +CbArrayView::GetHash() const +{ + IoHashStream Hash; + GetHash(Hash); + return Hash.GetHash(); +} + +void +CbArrayView::GetHash(IoHashStream& HashStream) const +{ + const CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType()); + HashStream.Append(&SerializedType, sizeof(SerializedType)); + auto _ = GetPayloadView(); + HashStream.Append(_.GetData(), _.GetSize()); +} + +bool +CbArrayView::Equals(const CbArrayView& Other) const +{ + return CbFieldTypeOps::GetType(GetType()) == CbFieldTypeOps::GetType(Other.GetType()) && + GetPayloadView().EqualBytes(Other.GetPayloadView()); +} + +void +CbArrayView::CopyTo(MutableMemoryView Buffer) const +{ + const MemoryView Source = GetPayloadView(); + ZEN_ASSERT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize()); + // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."), + // Buffer.GetSize(), + // sizeof(CbFieldType) + Source.GetSize()); + + *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetType(GetType()); + Buffer.RightChopInline(sizeof(CbFieldType)); + memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize()); +} + +void +CbArrayView::CopyTo(BinaryWriter& Ar) const +{ + const MemoryView Source = GetPayloadView(); + CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType()); + Ar.Write(&SerializedType, sizeof(SerializedType)); + Ar.Write(Source.GetData(), Source.GetSize()); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +CbObjectView::CbObjectView() : CbFieldView(usonprivate::GEmptyObjectPayload) +{ +} + +CbFieldViewIterator +CbObjectView::CreateViewIterator() const +{ + const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload()); + uint32_t PayloadSizeByteCount; + const uint64_t PayloadSize = ReadVarUInt(PayloadBytes, PayloadSizeByteCount); + + PayloadBytes += PayloadSizeByteCount; + + if (PayloadSize) + { + const void* const PayloadEnd = PayloadBytes + PayloadSize; + const CbFieldType UniformType = + CbFieldTypeOps::GetType(GetType()) == CbFieldType::UniformObject ? CbFieldType(*PayloadBytes++) : CbFieldType::HasFieldType; + return CbFieldViewIterator::MakeRange(MemoryView(PayloadBytes, PayloadEnd), UniformType); + } + + return CbFieldViewIterator(); +} + +void +CbObjectView::VisitFields(ICbVisitor&) +{ +} + +CbFieldView +CbObjectView::FindView(const std::string_view Name) const +{ + for (const CbFieldView Field : *this) + { + if (Name == Field.GetName()) + { + return Field; + } + } + return CbFieldView(); +} + +CbFieldView +CbObjectView::FindViewIgnoreCase(const std::string_view Name) const +{ + for (const CbFieldView Field : *this) + { + if (Name == Field.GetName()) + { + return Field; + } + } + return CbFieldView(); +} + +uint64_t +CbObjectView::GetSize() const +{ + return sizeof(CbFieldType) + GetPayloadSize(); +} + +IoHash +CbObjectView::GetHash() const +{ + IoHashStream Hash; + GetHash(Hash); + return Hash.GetHash(); +} + +void +CbObjectView::GetHash(IoHashStream& HashStream) const +{ + const CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType()); + HashStream.Append(&SerializedType, sizeof(SerializedType)); + HashStream.Append(GetPayloadView()); +} + +bool +CbObjectView::Equals(const CbObjectView& Other) const +{ + return CbFieldTypeOps::GetType(GetType()) == CbFieldTypeOps::GetType(Other.GetType()) && + GetPayloadView().EqualBytes(Other.GetPayloadView()); +} + +void +CbObjectView::CopyTo(MutableMemoryView Buffer) const +{ + const MemoryView Source = GetPayloadView(); + ZEN_ASSERT(Buffer.GetSize() == (sizeof(CbFieldType) + Source.GetSize())); + // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."), + // Buffer.GetSize(), + // sizeof(CbFieldType) + Source.GetSize()); + *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetType(GetType()); + Buffer.RightChopInline(sizeof(CbFieldType)); + memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize()); +} + +void +CbObjectView::CopyTo(BinaryWriter& Ar) const +{ + const MemoryView Source = GetPayloadView(); + CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType()); + Ar.Write(&SerializedType, sizeof(SerializedType)); + Ar.Write(Source.GetData(), Source.GetSize()); +} + +////////////////////////////////////////////////////////////////////////// + +template<typename FieldType> +uint64_t +TCbFieldIterator<FieldType>::GetRangeSize() const +{ + MemoryView View; + if (TryGetSerializedRangeView(View)) + { + return View.GetSize(); + } + else + { + uint64_t Size = 0; + for (CbFieldViewIterator It(*this); It; ++It) + { + Size += It.GetSize(); + } + return Size; + } +} + +template<typename FieldType> +IoHash +TCbFieldIterator<FieldType>::GetRangeHash() const +{ + IoHashStream Hash; + GetRangeHash(Hash); + return IoHash(Hash.GetHash()); +} + +template<typename FieldType> +void +TCbFieldIterator<FieldType>::GetRangeHash(IoHashStream& Hash) const +{ + MemoryView View; + if (TryGetSerializedRangeView(View)) + { + Hash.Append(View.GetData(), View.GetSize()); + } + else + { + for (CbFieldViewIterator It(*this); It; ++It) + { + It.GetHash(Hash); + } + } +} + +template<typename FieldType> +void +TCbFieldIterator<FieldType>::CopyRangeTo(MutableMemoryView InBuffer) const +{ + MemoryView Source; + if (TryGetSerializedRangeView(Source)) + { + ZEN_ASSERT(InBuffer.GetSize() == Source.GetSize()); + // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."), + // InBuffer.GetSize(), + // Source.GetSize()); + memcpy(InBuffer.GetData(), Source.GetData(), Source.GetSize()); + } + else + { + for (CbFieldViewIterator It(*this); It; ++It) + { + const uint64_t Size = It.GetSize(); + It.CopyTo(InBuffer.Left(Size)); + InBuffer.RightChopInline(Size); + } + } +} + +template class TCbFieldIterator<CbFieldView>; +template class TCbFieldIterator<CbField>; + +template<typename FieldType> +void +TCbFieldIterator<FieldType>::IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const +{ + if (CbFieldTypeOps::HasFieldType(FieldType::GetType())) + { + // Always iterate over non-uniform ranges because we do not know if they contain an attachment. + for (CbFieldViewIterator It(*this); It; ++It) + { + if (CbFieldTypeOps::MayContainAttachments(It.GetType())) + { + It.IterateAttachments(Visitor); + } + } + } + else + { + // Only iterate over uniform ranges if the uniform type may contain an attachment. + if (CbFieldTypeOps::MayContainAttachments(FieldType::GetType())) + { + for (CbFieldViewIterator It(*this); It; ++It) + { + It.IterateAttachments(Visitor); + } + } + } +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +CbFieldIterator +CbFieldIterator::CloneRange(const CbFieldViewIterator& It) +{ + MemoryView View; + if (It.TryGetSerializedRangeView(View)) + { + return MakeRange(SharedBuffer::Clone(View)); + } + else + { + UniqueBuffer Buffer = UniqueBuffer::Alloc(It.GetRangeSize()); + It.CopyRangeTo(MutableMemoryView(Buffer.GetData(), Buffer.GetSize())); + return MakeRange(SharedBuffer(std::move(Buffer))); + } +} + +SharedBuffer +CbFieldIterator::GetRangeBuffer() const +{ + const MemoryView RangeView = GetRangeView(); + const SharedBuffer& OuterBuffer = GetOuterBuffer(); + return OuterBuffer.GetView() == RangeView ? OuterBuffer : SharedBuffer::MakeView(RangeView, OuterBuffer); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +uint64_t +MeasureCompactBinary(MemoryView View, CbFieldType Type) +{ + uint64_t Size; + return TryMeasureCompactBinary(View, Type, Size, Type) ? Size : 0; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +bool +TryMeasureCompactBinary(MemoryView View, CbFieldType& OutType, uint64_t& OutSize, CbFieldType Type) +{ + uint64_t Size = 0; + + if (CbFieldTypeOps::HasFieldType(Type)) + { + if (View.GetSize() == 0) + { + OutType = CbFieldType::None; + OutSize = 1; + return false; + } + + Type = *static_cast<const CbFieldType*>(View.GetData()); + View.RightChopInline(1); + Size += 1; + } + + bool bDynamicSize = false; + uint64_t FixedSize = 0; + switch (CbFieldTypeOps::GetType(Type)) + { + case CbFieldType::Null: + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + case CbFieldType::Array: + case CbFieldType::UniformArray: + case CbFieldType::Binary: + case CbFieldType::String: + case CbFieldType::IntegerPositive: + case CbFieldType::IntegerNegative: + bDynamicSize = true; + break; + case CbFieldType::Float32: + FixedSize = 4; + break; + case CbFieldType::Float64: + FixedSize = 8; + break; + case CbFieldType::BoolFalse: + case CbFieldType::BoolTrue: + break; + case CbFieldType::CompactBinaryAttachment: + case CbFieldType::BinaryAttachment: + case CbFieldType::Hash: + FixedSize = 20; + break; + case CbFieldType::Uuid: + FixedSize = 16; + break; + case CbFieldType::ObjectId: + FixedSize = 12; + break; + case CbFieldType::DateTime: + case CbFieldType::TimeSpan: + FixedSize = 8; + break; + case CbFieldType::None: + default: + OutType = CbFieldType::None; + OutSize = 0; + return false; + } + + OutType = Type; + + if (CbFieldTypeOps::HasFieldName(Type)) + { + if (View.GetSize() == 0) + { + OutSize = Size + 1; + return false; + } + + uint32_t NameLenByteCount = MeasureVarUInt(View.GetData()); + if (View.GetSize() < NameLenByteCount) + { + OutSize = Size + NameLenByteCount; + return false; + } + + const uint64_t NameLen = ReadVarUInt(View.GetData(), NameLenByteCount); + const uint64_t NameSize = NameLen + NameLenByteCount; + + if (bDynamicSize && View.GetSize() < NameSize) + { + OutSize = Size + NameSize; + return false; + } + + View.RightChopInline(NameSize); + Size += NameSize; + } + + switch (CbFieldTypeOps::GetType(Type)) + { + case CbFieldType::Object: + case CbFieldType::UniformObject: + case CbFieldType::Array: + case CbFieldType::UniformArray: + case CbFieldType::Binary: + case CbFieldType::String: + if (View.GetSize() == 0) + { + OutSize = Size + 1; + return false; + } + else + { + uint32_t PayloadSizeByteCount = MeasureVarUInt(View.GetData()); + if (View.GetSize() < PayloadSizeByteCount) + { + OutSize = Size + PayloadSizeByteCount; + return false; + } + const uint64_t PayloadSize = ReadVarUInt(View.GetData(), PayloadSizeByteCount); + OutSize = Size + PayloadSize + PayloadSizeByteCount; + } + return true; + + case CbFieldType::IntegerPositive: + case CbFieldType::IntegerNegative: + if (View.GetSize() == 0) + { + OutSize = Size + 1; + return false; + } + OutSize = Size + MeasureVarUInt(View.GetData()); + return true; + + default: + OutSize = Size + FixedSize; + return true; + } +} + +////////////////////////////////////////////////////////////////////////// + +CbField +LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) +{ + std::vector<uint8_t> HeaderBytes; + CbFieldType FieldType; + uint64_t FieldSize = 1; + + // Read in small increments until the total field size is known, to avoid reading too far. + for (;;) + { + const int32_t ReadSize = int32_t(FieldSize - HeaderBytes.size()); + const size_t ReadOffset = HeaderBytes.size(); + HeaderBytes.resize(ReadOffset + ReadSize); + + Ar.Read(HeaderBytes.data() + ReadOffset, ReadSize); + if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes), FieldType, FieldSize)) + { + break; + } + ZEN_ASSERT(FieldSize > 0, "Failed to load from invalid compact binary data."); + } + + // Allocate the buffer, copy the header, and read the remainder of the field. + UniqueBuffer Buffer = Allocator(FieldSize); + ZEN_ASSERT(Buffer.GetSize() == FieldSize); + MutableMemoryView View = Buffer.GetView(); + memcpy(View.GetData(), HeaderBytes.data(), HeaderBytes.size()); + View.RightChopInline(HeaderBytes.size()); + if (!View.IsEmpty()) + { + Ar.Read(View.GetData(), View.GetSize()); + } + return CbField(SharedBuffer(std::move(Buffer))); +} + +////////////////////////////////////////////////////////////////////////// + +void +SaveCompactBinary(BinaryWriter& Ar, const CbFieldView& Field) +{ + Field.CopyTo(Ar); +} + +void +SaveCompactBinary(BinaryWriter& Ar, const CbArrayView& Array) +{ + Array.CopyTo(Ar); +} + +void +SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object) +{ + Object.CopyTo(Ar); +} + +////////////////////////////////////////////////////////////////////////// + +StringBuilderBase& +ToString(CbObjectView& Root, StringBuilderBase& OutString) +{ + ryml::Tree Tree; + + ryml::NodeRef r = Tree.rootref(); + r |= ryml::MAP; + + for (CbFieldViewIterator It = Root.CreateViewIterator(); It; ++It) + { + } + + return OutString; +} + +////////////////////////////////////////////////////////////////////////// + +void +uson_forcelink() +{ +} + +TEST_CASE("uson") +{ + using namespace std::literals; + + SUBCASE("CbField") + { + constexpr CbFieldView DefaultField; + static_assert(!DefaultField.HasName(), "Error in HasName()"); + static_assert(!DefaultField.HasValue(), "Error in HasValue()"); + static_assert(!DefaultField.HasError(), "Error in HasError()"); + static_assert(DefaultField.GetError() == CbFieldError::None, "Error in GetError()"); + + CHECK(DefaultField.GetSize() == 1); + CHECK(DefaultField.GetName().size() == 0); + CHECK(DefaultField.HasName() == false); + CHECK(DefaultField.HasValue() == false); + CHECK(DefaultField.HasError() == false); + CHECK(DefaultField.GetError() == CbFieldError::None); + + const uint8_t Type = (uint8_t)CbFieldType::None; + CHECK(DefaultField.GetHash() == IoHash::HashMemory(&Type, sizeof Type)); + + CHECK(DefaultField.GetView() == MemoryView{}); + MemoryView SerializedView; + CHECK(DefaultField.TryGetSerializedView(SerializedView) == false); + } + + SUBCASE("CbField(None)") + { + CbFieldView NoneField(nullptr, CbFieldType::None); + CHECK(NoneField.GetSize() == 1); + CHECK(NoneField.GetName().size() == 0); + CHECK(NoneField.HasName() == false); + CHECK(NoneField.HasValue() == false); + CHECK(NoneField.HasError() == false); + CHECK(NoneField.GetError() == CbFieldError::None); + CHECK(NoneField.GetHash() == CbFieldView().GetHash()); + CHECK(NoneField.GetView() == MemoryView()); + MemoryView SerializedView; + CHECK(NoneField.TryGetSerializedView(SerializedView) == false); + } + + SUBCASE("CbField(None|Type|Name)") + { + constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; + constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; + CbFieldView NoneField(NoneBytes); + + CHECK(NoneField.GetSize() == sizeof(NoneBytes)); + CHECK(NoneField.GetName().compare("Name"sv) == 0); + CHECK(NoneField.HasName() == true); + CHECK(NoneField.HasValue() == false); + CHECK(NoneField.GetHash() == IoHash::HashMemory(NoneBytes, sizeof NoneBytes)); + CHECK(NoneField.GetView() == MemoryView(NoneBytes, sizeof NoneBytes)); + MemoryView SerializedView; + CHECK(NoneField.TryGetSerializedView(SerializedView) == true); + CHECK(SerializedView == MemoryView(NoneBytes, sizeof NoneBytes)); + + uint8_t CopyBytes[sizeof(NoneBytes)]; + NoneField.CopyTo(MutableMemoryView(CopyBytes, sizeof CopyBytes)); + CHECK(MemoryView(NoneBytes, sizeof NoneBytes).EqualBytes(MemoryView(CopyBytes, sizeof CopyBytes))); + } + + SUBCASE("CbField(None|Type)") + { + constexpr CbFieldType FieldType = CbFieldType::None; + constexpr const char NoneBytes[] = {char(FieldType)}; + CbFieldView NoneField(NoneBytes); + + CHECK(NoneField.GetSize() == sizeof NoneBytes); + CHECK(NoneField.GetName().size() == 0); + CHECK(NoneField.HasName() == false); + CHECK(NoneField.HasValue() == false); + CHECK(NoneField.GetHash() == CbFieldView().GetHash()); + CHECK(NoneField.GetView() == MemoryView(NoneBytes, sizeof NoneBytes)); + MemoryView SerializedView; + CHECK(NoneField.TryGetSerializedView(SerializedView) == true); + CHECK(SerializedView == MemoryView(NoneBytes, sizeof NoneBytes)); + } + + SUBCASE("CbField(None|Name)") + { + constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; + constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; + CbFieldView NoneField(NoneBytes + 1, FieldType); + CHECK(NoneField.GetSize() == uint64_t(sizeof NoneBytes)); + CHECK(NoneField.GetName().compare("Name") == 0); + CHECK(NoneField.HasName() == true); + CHECK(NoneField.HasValue() == false); + CHECK(NoneField.GetHash() == IoHash::HashMemory(NoneBytes, sizeof NoneBytes)); + CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1)); + MemoryView SerializedView; + CHECK(NoneField.TryGetSerializedView(SerializedView) == false); + + uint8_t CopyBytes[sizeof(NoneBytes)]; + NoneField.CopyTo(MutableMemoryView(CopyBytes, sizeof CopyBytes)); + CHECK(MemoryView(NoneBytes, sizeof NoneBytes).EqualBytes(MemoryView(CopyBytes, sizeof CopyBytes))); + } + + SUBCASE("CbField(None|EmptyName)") + { + constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; + constexpr const uint8_t NoneBytes[] = {uint8_t(FieldType), 0}; + CbFieldView NoneField(NoneBytes + 1, FieldType); + CHECK(NoneField.GetSize() == sizeof NoneBytes); + CHECK(NoneField.GetName().empty() == true); + CHECK(NoneField.HasName() == true); + CHECK(NoneField.HasValue() == false); + CHECK(NoneField.GetHash() == IoHash::HashMemory(NoneBytes, sizeof NoneBytes)); + CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1)); + MemoryView SerializedView; + CHECK(NoneField.TryGetSerializedView(SerializedView) == false); + } + + static_assert(!std::is_constructible<CbFieldView, const CbObjectView&>::value, "Invalid constructor for CbField"); + static_assert(!std::is_assignable<CbFieldView, const CbObjectView&>::value, "Invalid assignment for CbField"); + static_assert(!std::is_convertible<CbFieldView, CbObjectView>::value, "Invalid conversion to CbObject"); + static_assert(!std::is_assignable<CbObjectView, const CbFieldView&>::value, "Invalid assignment for CbObject"); +} + +TEST_CASE("uson.null") +{ + using namespace std::literals; + + SUBCASE("CbField(Null)") + { + CbFieldView NullField(nullptr, CbFieldType::Null); + CHECK(NullField.GetSize() == 1); + CHECK(NullField.IsNull() == true); + CHECK(NullField.HasValue() == true); + CHECK(NullField.HasError() == false); + CHECK(NullField.GetError() == CbFieldError::None); + const uint8_t Null[]{uint8_t(CbFieldType::Null)}; + CHECK(NullField.GetHash() == IoHash::HashMemory(Null, sizeof Null)); + } + + SUBCASE("CbField(None)") + { + CbFieldView Field; + CHECK(Field.IsNull() == false); + } +} + +} // namespace zen diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp new file mode 100644 index 000000000..d1422e5a2 --- /dev/null +++ b/zencore/compactbinarybuilder.cpp @@ -0,0 +1,1530 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/compactbinarybuilder.h" + +#include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/endian.h> +#include <zencore/stream.h> +#include <zencore/string.h> + +#define _USE_MATH_DEFINES +#include <math.h> + +#include <doctest/doctest.h> + +namespace zen { + +template<typename T> +uint64_t +AddUninitialized(std::vector<T>& Vector, uint64_t Count) +{ + const uint64_t Offset = Vector.size(); + Vector.resize(Offset + Count); + return Offset; +} + +template<typename T> +uint64_t +Append(std::vector<T>& Vector, const T* Data, uint64_t Count) +{ + const uint64_t Offset = Vector.size(); + Vector.resize(Offset + Count); + + memcpy(Vector.data() + Offset, Data, sizeof(T) * Count); + + return Offset; +} + +////////////////////////////////////////////////////////////////////////// + +enum class CbWriter::StateFlags : uint8_t +{ + None = 0, + /** Whether a name has been written for the current field. */ + Name = 1 << 0, + /** Whether this state is in the process of writing a field. */ + Field = 1 << 1, + /** Whether this state is for array fields. */ + Array = 1 << 2, + /** Whether this state is for object fields. */ + Object = 1 << 3, +}; + +ENUM_CLASS_FLAGS(CbWriter::StateFlags); + +/** Whether the field type can be used in a uniform array or uniform object. */ +static constexpr bool +IsUniformType(const CbFieldType Type) +{ + if (CbFieldTypeOps::HasFieldName(Type)) + { + return true; + } + + switch (Type) + { + case CbFieldType::None: + case CbFieldType::Null: + case CbFieldType::BoolFalse: + case CbFieldType::BoolTrue: + return false; + default: + return true; + } +} + +/** Append the payload from the compact binary value to the array and return its type. */ +static inline CbFieldType +AppendCompactBinary(const CbFieldView& Value, std::vector<uint8_t>& OutData) +{ + struct FCopy : public CbFieldView + { + using CbFieldView::GetPayloadView; + using CbFieldView::GetType; + }; + const FCopy& ValueCopy = static_cast<const FCopy&>(Value); + const MemoryView SourceView = ValueCopy.GetPayloadView(); + const uint64_t TargetOffset = OutData.size(); + OutData.resize(TargetOffset + SourceView.GetSize()); + memcpy(OutData.data() + TargetOffset, SourceView.GetData(), SourceView.GetSize()); + return CbFieldTypeOps::GetType(ValueCopy.GetType()); +} + +CbWriter::CbWriter() +{ + States.emplace_back(); +} + +CbWriter::CbWriter(const int64_t InitialSize) : CbWriter() +{ + Data.reserve(InitialSize); +} + +CbWriter::~CbWriter() +{ +} + +void +CbWriter::Reset() +{ + Data.resize(0); + States.resize(0); + States.emplace_back(); +} + +CbFieldIterator +CbWriter::Save() +{ + const uint64_t Size = GetSaveSize(); + UniqueBuffer Buffer = UniqueBuffer::Alloc(Size); + const CbFieldViewIterator Output = Save(MutableMemoryView(Buffer.GetData(), Buffer.GetSize())); + SharedBuffer SharedBuf(std::move(Buffer)); + + return CbFieldIterator::MakeRangeView(Output, SharedBuf); +} + +CbFieldViewIterator +CbWriter::Save(const MutableMemoryView Buffer) +{ + ZEN_ASSERT(States.size() == 1 && States.back().Flags == StateFlags::None); + // TEXT("It is invalid to save while there are incomplete write operations.")); + ZEN_ASSERT(Data.size() > 0); // TEXT("It is invalid to save when nothing has been written.")); + ZEN_ASSERT(Buffer.GetSize() == Data.size()); + // TEXT("Buffer is %" UINT64_FMT " bytes but %" INT64_FMT " is required."), + // Buffer.GetSize(), + // Data.Num()); + memcpy(Buffer.GetData(), Data.data(), Data.size()); + return CbFieldViewIterator::MakeRange(Buffer); +} + +void +CbWriter::Save(BinaryWriter& Writer) +{ + ZEN_ASSERT(States.size() == 1 && States.back().Flags == StateFlags::None); + // TEXT("It is invalid to save while there are incomplete write operations.")); + ZEN_ASSERT(Data.size() > 0); // TEXT("It is invalid to save when nothing has been written.")); + Writer.Write(Data.data(), Data.size()); +} + +uint64_t +CbWriter::GetSaveSize() const +{ + return Data.size(); +} + +void +CbWriter::BeginField() +{ + WriterState& State = States.back(); + if ((State.Flags & StateFlags::Field) == StateFlags::None) + { + State.Flags |= StateFlags::Field; + State.Offset = Data.size(); + Data.push_back(0); + } + else + { + ZEN_ASSERT((State.Flags & StateFlags::Name) == StateFlags::Name); + // TEXT("A new field cannot be written until the previous field '%.*hs' is finished."), + // GetActiveName().Len(), + // GetActiveName().GetData()); + } +} + +void +CbWriter::EndField(CbFieldType Type) +{ + WriterState& State = States.back(); + + if ((State.Flags & StateFlags::Name) == StateFlags::Name) + { + Type |= CbFieldType::HasFieldName; + } + else + { + ZEN_ASSERT((State.Flags & StateFlags::Object) == StateFlags::None); + // TEXT("It is invalid to write an object field without a unique non-empty name.")); + } + + if (State.Count == 0) + { + State.UniformType = Type; + } + else if (State.UniformType != Type) + { + State.UniformType = CbFieldType::None; + } + + State.Flags &= ~(StateFlags::Name | StateFlags::Field); + ++State.Count; + Data[State.Offset] = uint8_t(Type); +} + +ZEN_NOINLINE +CbWriter& +CbWriter::SetName(const std::string_view Name) +{ + WriterState& State = States.back(); + ZEN_ASSERT((State.Flags & StateFlags::Array) != StateFlags::Array); + // TEXT("It is invalid to write a name for an array field. Name '%.*hs'"), + // Name.Len(), + // Name.GetData()); + ZEN_ASSERT(!Name.empty()); + // TEXT("%s"), + //(State.Flags & EStateFlags::Object) == EStateFlags::Object + // ? TEXT("It is invalid to write an empty name for an object field. Specify a unique non-empty name.") + // : TEXT("It is invalid to write an empty name for a top-level field. Specify a name or avoid this call.")); + ZEN_ASSERT((State.Flags & (StateFlags::Name | StateFlags::Field)) == StateFlags::None); + // TEXT("A new field '%.*hs' cannot be written until the previous field '%.*hs' is finished."), + // Name.Len(), + // Name.GetData(), + // GetActiveName().Len(), + // GetActiveName().GetData()); + + BeginField(); + State.Flags |= StateFlags::Name; + const uint32_t NameLenByteCount = MeasureVarUInt(uint32_t(Name.size())); + const int64_t NameLenOffset = Data.size(); + Data.resize(NameLenOffset + NameLenByteCount); + + WriteVarUInt(uint64_t(Name.size()), Data.data() + NameLenOffset); + + const uint8_t* NamePtr = reinterpret_cast<const uint8_t*>(Name.data()); + Data.insert(Data.end(), NamePtr, NamePtr + Name.size()); + return *this; +} + +void +CbWriter::SetNameOrAddString(const std::string_view NameOrValue) +{ + // A name is only written if it would begin a new field inside of an object. + if ((States.back().Flags & (StateFlags::Name | StateFlags::Field | StateFlags::Object)) == StateFlags::Object) + { + SetName(NameOrValue); + } + else + { + AddString(NameOrValue); + } +} + +std::string_view +CbWriter::GetActiveName() const +{ + const WriterState& State = States.back(); + if ((State.Flags & StateFlags::Name) == StateFlags::Name) + { + const uint8_t* const EncodedName = Data.data() + State.Offset + sizeof(CbFieldType); + uint32_t NameLenByteCount; + const uint64_t NameLen = ReadVarUInt(EncodedName, NameLenByteCount); + const size_t ClampedNameLen = std::clamp<uint64_t>(NameLen, 0, ~uint64_t(0)); + return std::string_view(reinterpret_cast<const char*>(EncodedName + NameLenByteCount), ClampedNameLen); + } + return std::string_view(); +} + +void +CbWriter::MakeFieldsUniform(const int64_t FieldBeginOffset, const int64_t FieldEndOffset) +{ + MutableMemoryView SourceView(Data.data() + FieldBeginOffset, uint64_t(FieldEndOffset - FieldBeginOffset)); + MutableMemoryView TargetView = SourceView; + TargetView.RightChopInline(sizeof(CbFieldType)); + + while (!SourceView.IsEmpty()) + { + const uint64_t FieldSize = MeasureCompactBinary(SourceView) - sizeof(CbFieldType); + SourceView.RightChopInline(sizeof(CbFieldType)); + if (TargetView.GetData() != SourceView.GetData()) + { + memmove(TargetView.GetData(), SourceView.GetData(), FieldSize); + } + SourceView.RightChopInline(FieldSize); + TargetView.RightChopInline(FieldSize); + } + + if (!TargetView.IsEmpty()) + { + const auto EraseBegin = Data.begin() + (FieldEndOffset - TargetView.GetSize()); + const auto EraseEnd = EraseBegin + TargetView.GetSize(); + + Data.erase(EraseBegin, EraseEnd); + } +} + +void +CbWriter::AddField(const CbFieldView& Value) +{ + ZEN_ASSERT(Value.HasValue()); // , TEXT("It is invalid to write a field with no value.")); + BeginField(); + EndField(AppendCompactBinary(Value, Data)); +} + +void +CbWriter::AddField(const CbField& Value) +{ + AddField(CbFieldView(Value)); +} + +void +CbWriter::BeginObject() +{ + BeginField(); + States.push_back(WriterState()); + States.back().Flags |= StateFlags::Object; +} + +void +CbWriter::EndObject() +{ + ZEN_ASSERT(States.size() > 1 && (States.back().Flags & StateFlags::Object) == StateFlags::Object); + + // TEXT("It is invalid to end an object when an object is not at the top of the stack.")); + ZEN_ASSERT((States.back().Flags & StateFlags::Field) == StateFlags::None); + // TEXT("It is invalid to end an object until the previous field is finished.")); + + const bool bUniform = IsUniformType(States.back().UniformType); + const uint64_t Count = States.back().Count; + States.pop_back(); + + // Calculate the offset of the payload. + const WriterState& State = States.back(); + int64_t PayloadOffset = State.Offset + 1; + if ((State.Flags & StateFlags::Name) == StateFlags::Name) + { + uint32_t NameLenByteCount; + const uint64_t NameLen = ReadVarUInt(Data.data() + PayloadOffset, NameLenByteCount); + PayloadOffset += NameLen + NameLenByteCount; + } + + // Remove redundant field types for uniform objects. + if (bUniform && Count > 1) + { + MakeFieldsUniform(PayloadOffset, Data.size()); + } + + // Insert the object size. + const uint64_t Size = uint64_t(Data.size() - PayloadOffset); + const uint32_t SizeByteCount = MeasureVarUInt(Size); + Data.insert(Data.begin() + PayloadOffset, SizeByteCount, 0); + WriteVarUInt(Size, Data.data() + PayloadOffset); + + EndField(bUniform ? CbFieldType::UniformObject : CbFieldType::Object); +} + +void +CbWriter::AddObject(const CbObjectView& Value) +{ + BeginField(); + EndField(AppendCompactBinary(Value.AsFieldView(), Data)); +} + +void +CbWriter::AddObject(const CbObject& Value) +{ + AddObject(CbObjectView(Value)); +} + +ZEN_NOINLINE +void +CbWriter::BeginArray() +{ + BeginField(); + States.push_back(WriterState()); + States.back().Flags |= StateFlags::Array; +} + +void +CbWriter::EndArray() +{ + ZEN_ASSERT(States.size() > 1 && (States.back().Flags & StateFlags::Array) == StateFlags::Array); + // TEXT("Invalid attempt to end an array when an array is not at the top of the stack.")); + ZEN_ASSERT((States.back().Flags & StateFlags::Field) == StateFlags::None); + // TEXT("It is invalid to end an array until the previous field is finished.")); + const bool bUniform = IsUniformType(States.back().UniformType); + const uint64_t Count = States.back().Count; + States.pop_back(); + + // Calculate the offset of the payload. + const WriterState& State = States.back(); + int64_t PayloadOffset = State.Offset + 1; + if ((State.Flags & StateFlags::Name) == StateFlags::Name) + { + uint32_t NameLenByteCount; + const uint64_t NameLen = ReadVarUInt(Data.data() + PayloadOffset, NameLenByteCount); + PayloadOffset += NameLen + NameLenByteCount; + } + + // Remove redundant field types for uniform arrays. + if (bUniform && Count > 1) + { + MakeFieldsUniform(PayloadOffset, Data.size()); + } + + // Insert the array size and field count. + const uint32_t CountByteCount = MeasureVarUInt(Count); + const uint64_t Size = uint64_t(Data.size() - PayloadOffset) + CountByteCount; + const uint32_t SizeByteCount = MeasureVarUInt(Size); + Data.insert(Data.begin() + PayloadOffset, SizeByteCount + CountByteCount, 0); + WriteVarUInt(Size, Data.data() + PayloadOffset); + WriteVarUInt(Count, Data.data() + PayloadOffset + SizeByteCount); + + EndField(bUniform ? CbFieldType::UniformArray : CbFieldType::Array); +} + +void +CbWriter::AddArray(const CbArrayView& Value) +{ + BeginField(); + EndField(AppendCompactBinary(Value.AsFieldView(), Data)); +} + +void +CbWriter::AddArray(const CbArray& Value) +{ + AddArray(CbArrayView(Value)); +} + +void +CbWriter::AddNull() +{ + BeginField(); + EndField(CbFieldType::Null); +} + +void +CbWriter::AddBinary(const void* const Value, const uint64_t Size) +{ + BeginField(); + const uint32_t SizeByteCount = MeasureVarUInt(Size); + const int64_t SizeOffset = Data.size(); + Data.resize(Data.size() + SizeByteCount); + WriteVarUInt(Size, Data.data() + SizeOffset); + Data.insert(Data.end(), static_cast<const uint8_t*>(Value), static_cast<const uint8_t*>(Value) + Size); + EndField(CbFieldType::Binary); +} + +void +CbWriter::AddBinary(IoBuffer Buffer) +{ + AddBinary(Buffer.Data(), Buffer.Size()); +} + +void +CbWriter::AddBinary(SharedBuffer Buffer) +{ + AddBinary(Buffer.GetData(), Buffer.GetSize()); +} + +void +CbWriter::AddString(const std::string_view Value) +{ + BeginField(); + const uint64_t Size = uint64_t(Value.size()); + const uint32_t SizeByteCount = MeasureVarUInt(Size); + const int64_t Offset = Data.size(); + + Data.resize(Offset + SizeByteCount + Size); + + uint8_t* StringData = Data.data() + Offset; + WriteVarUInt(Size, StringData); + StringData += SizeByteCount; + if (Size > 0) + { + memcpy(StringData, Value.data(), Value.size() * sizeof(char)); + } + EndField(CbFieldType::String); +} + +void +CbWriter::AddString(const std::wstring_view Value) +{ + BeginField(); + ExtendableStringBuilder<128> Utf8; + WideToUtf8(Value, Utf8); + + const uint32_t Size = uint32_t(Utf8.Size()); + const uint32_t SizeByteCount = MeasureVarUInt(Size); + const int64_t Offset = Data.size(); + Data.resize(Offset + SizeByteCount + Size); + uint8_t* StringData = Data.data() + Offset; + WriteVarUInt(Size, StringData); + StringData += SizeByteCount; + if (Size > 0) + { + memcpy(reinterpret_cast<char*>(StringData), Utf8.Data(), Utf8.Size()); + } + EndField(CbFieldType::String); +} + +ZEN_NOINLINE +void +CbWriter::AddInteger(const int32_t Value) +{ + if (Value >= 0) + { + return AddInteger(uint32_t(Value)); + } + BeginField(); + const uint32_t Magnitude = ~uint32_t(Value); + const uint32_t MagnitudeByteCount = MeasureVarUInt(Magnitude); + const int64_t Offset = Data.size(); + Data.resize(Offset + MagnitudeByteCount); + WriteVarUInt(Magnitude, Data.data() + Offset); + EndField(CbFieldType::IntegerNegative); +} + +void +CbWriter::AddInteger(const int64_t Value) +{ + if (Value >= 0) + { + return AddInteger(uint64_t(Value)); + } + BeginField(); + const uint64_t Magnitude = ~uint64_t(Value); + const uint32_t MagnitudeByteCount = MeasureVarUInt(Magnitude); + const uint64_t Offset = AddUninitialized(Data, MagnitudeByteCount); + WriteVarUInt(Magnitude, Data.data() + Offset); + EndField(CbFieldType::IntegerNegative); +} + +ZEN_NOINLINE +void +CbWriter::AddInteger(const uint32_t Value) +{ + BeginField(); + const uint32_t ValueByteCount = MeasureVarUInt(Value); + const uint64_t Offset = AddUninitialized(Data, ValueByteCount); + WriteVarUInt(Value, Data.data() + Offset); + EndField(CbFieldType::IntegerPositive); +} + +ZEN_NOINLINE +void +CbWriter::AddInteger(const uint64_t Value) +{ + BeginField(); + const uint32_t ValueByteCount = MeasureVarUInt(Value); + const uint64_t Offset = AddUninitialized(Data, ValueByteCount); + WriteVarUInt(Value, Data.data() + Offset); + EndField(CbFieldType::IntegerPositive); +} + +ZEN_NOINLINE +void +CbWriter::AddFloat(const float Value) +{ + BeginField(); + const uint32_t RawValue = FromNetworkOrder(reinterpret_cast<const uint32_t&>(Value)); + Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint32_t)); + EndField(CbFieldType::Float32); +} + +ZEN_NOINLINE +void +CbWriter::AddFloat(const double Value) +{ + const float Value32 = float(Value); + if (Value == double(Value32)) + { + return AddFloat(Value32); + } + BeginField(); + const uint64_t RawValue = FromNetworkOrder(reinterpret_cast<const uint64_t&>(Value)); + Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t)); + EndField(CbFieldType::Float64); +} + +ZEN_NOINLINE +void +CbWriter::AddBool(const bool bValue) +{ + BeginField(); + EndField(bValue ? CbFieldType::BoolTrue : CbFieldType::BoolFalse); +} + +ZEN_NOINLINE +void +CbWriter::AddCompactBinaryAttachment(const IoHash& Value) +{ + BeginField(); + Append(Data, Value.Hash, sizeof Value.Hash); + EndField(CbFieldType::CompactBinaryAttachment); +} + +ZEN_NOINLINE +void +CbWriter::AddBinaryAttachment(const IoHash& Value) +{ + BeginField(); + Append(Data, Value.Hash, sizeof Value.Hash); + EndField(CbFieldType::BinaryAttachment); +} + +ZEN_NOINLINE +void +CbWriter::AddAttachment(const CbAttachment& Attachment) +{ + BeginField(); + const IoHash& Value = Attachment.GetHash(); + Append(Data, Value.Hash, sizeof Value.Hash); + EndField(CbFieldType::BinaryAttachment); +} + +ZEN_NOINLINE +void +CbWriter::AddHash(const IoHash& Value) +{ + BeginField(); + Append(Data, Value.Hash, sizeof Value.Hash); + EndField(CbFieldType::Hash); +} + +void +CbWriter::AddUuid(const Guid& Value) +{ + const auto AppendSwappedBytes = [this](uint32_t In) { + In = FromNetworkOrder(In); + Append(Data, reinterpret_cast<const uint8_t*>(&In), sizeof In); + }; + BeginField(); + AppendSwappedBytes(Value.A); + AppendSwappedBytes(Value.B); + AppendSwappedBytes(Value.C); + AppendSwappedBytes(Value.D); + EndField(CbFieldType::Uuid); +} + +void +CbWriter::AddObjectId(const Oid& Value) +{ + BeginField(); + Append(Data, reinterpret_cast<const uint8_t*>(&Value.OidBits), sizeof Value.OidBits); + EndField(CbFieldType::ObjectId); +} + +void +CbWriter::AddDateTimeTicks(const int64_t Ticks) +{ + BeginField(); + const uint64_t RawValue = FromNetworkOrder(uint64_t(Ticks)); + Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t)); + EndField(CbFieldType::DateTime); +} + +void +CbWriter::AddDateTime(const DateTime Value) +{ + AddDateTimeTicks(Value.GetTicks()); +} + +void +CbWriter::AddTimeSpanTicks(const int64_t Ticks) +{ + BeginField(); + const uint64_t RawValue = FromNetworkOrder(uint64_t(Ticks)); + Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t)); + EndField(CbFieldType::TimeSpan); +} + +void +CbWriter::AddTimeSpan(const TimeSpan Value) +{ + AddTimeSpanTicks(Value.GetTicks()); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +CbWriter& +operator<<(CbWriter& Writer, const DateTime Value) +{ + Writer.AddDateTime(Value); + return Writer; +} + +CbWriter& +operator<<(CbWriter& Writer, const TimeSpan Value) +{ + Writer.AddTimeSpan(Value); + return Writer; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void +usonbuilder_forcelink() +{ +} + +doctest::String +toString(const DateTime&) +{ + // TODO:implement + return ""; +} + +doctest::String +toString(const TimeSpan&) +{ + // TODO:implement + return ""; +} + +TEST_CASE("usonbuilder.object") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + SUBCASE("EmptyObject") + { + Writer.BeginObject(); + Writer.EndObject(); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.IsObject() == true); + CHECK(Field.AsObjectView().CreateViewIterator().HasValue() == false); + } + + SUBCASE("NamedEmptyObject") + { + Writer.SetName("Object"sv); + Writer.BeginObject(); + Writer.EndObject(); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.IsObject() == true); + CHECK(Field.AsObjectView().CreateViewIterator().HasValue() == false); + } + + SUBCASE("BasicObject") + { + Writer.BeginObject(); + Writer.SetName("Integer"sv).AddInteger(0); + Writer.SetName("Float"sv).AddFloat(0.0f); + Writer.EndObject(); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.IsObject() == true); + + CbObjectView Object = Field.AsObjectView(); + CHECK(Object["Integer"sv].IsInteger() == true); + CHECK(Object["Float"sv].IsFloat() == true); + } + + SUBCASE("UniformObject") + { + Writer.BeginObject(); + Writer.SetName("Field1"sv).AddInteger(0); + Writer.SetName("Field2"sv).AddInteger(1); + Writer.EndObject(); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.IsObject() == true); + + CbObjectView Object = Field.AsObjectView(); + CHECK(Object["Field1"sv].IsInteger() == true); + CHECK(Object["Field2"sv].IsInteger() == true); + } +} + +TEST_CASE("usonbuilder.array") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + SUBCASE("EmptyArray") + { + Writer.BeginArray(); + Writer.EndArray(); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.IsArray() == true); + CHECK(Field.AsArrayView().Num() == 0); + } + + SUBCASE("NamedEmptyArray") + { + Writer.SetName("Array"sv); + Writer.BeginArray(); + Writer.EndArray(); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.IsArray() == true); + CHECK(Field.AsArrayView().Num() == 0); + } + + SUBCASE("BasicArray") + { + Writer.BeginArray(); + Writer.AddInteger(0); + Writer.AddFloat(0.0f); + Writer.EndArray(); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.IsArray() == true); + CbFieldViewIterator Iterator = Field.AsArrayView().CreateViewIterator(); + CHECK(Iterator.IsInteger() == true); + ++Iterator; + CHECK(Iterator.IsFloat() == true); + ++Iterator; + CHECK(Iterator.HasValue() == false); + } + + SUBCASE("UniformArray") + { + Writer.BeginArray(); + Writer.AddInteger(0); + Writer.AddInteger(1); + Writer.EndArray(); + + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.IsArray() == true); + CbFieldViewIterator Iterator = Field.AsArrayView().CreateViewIterator(); + CHECK(Iterator.IsInteger() == true); + ++Iterator; + CHECK(Iterator.IsInteger() == true); + ++Iterator; + CHECK(Iterator.HasValue() == false); + } +} + +TEST_CASE("usonbuilder.null") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + SUBCASE("Null") + { + Writer.AddNull(); + CbField Field = Writer.Save(); + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.HasName() == false); + CHECK(Field.IsNull() == true); + } + + SUBCASE("NullWithName") + { + Writer.SetName("Null"sv); + Writer.AddNull(); + CbField Field = Writer.Save(); + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + CHECK(Field.HasName() == true); + CHECK(Field.GetName().compare("Null"sv) == 0); + CHECK(Field.IsNull() == true); + } + + SUBCASE("Null Array/Object Uniformity") + { + Writer.BeginArray(); + Writer.AddNull(); + Writer.AddNull(); + Writer.AddNull(); + Writer.EndArray(); + + Writer.BeginObject(); + Writer.SetName("N1"sv).AddNull(); + Writer.SetName("N2"sv).AddNull(); + Writer.SetName("N3"sv).AddNull(); + Writer.EndObject(); + + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + } + + SUBCASE("Null with Save(Buffer)") + { + constexpr int NullCount = 3; + for (int Index = 0; Index < NullCount; ++Index) + { + Writer.AddNull(); + } + uint8_t Buffer[NullCount]{}; + MutableMemoryView BufferView(Buffer, sizeof Buffer); + CbFieldViewIterator Fields = Writer.Save(BufferView); + + CHECK(ValidateCompactBinaryRange(BufferView, CbValidateMode::All) == CbValidateError::None); + + for (int Index = 0; Index < NullCount; ++Index) + { + CHECK(Fields.IsNull() == true); + ++Fields; + } + CHECK(Fields.HasValue() == false); + } +} + +TEST_CASE("usonbuilder.binary") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; +} + +TEST_CASE("usonbuilder.string") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + SUBCASE("Empty Strings") + { + Writer.AddString(std::string_view()); + Writer.AddString(std::wstring_view()); + + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + for (CbFieldView Field : Fields) + { + CHECK(Field.HasName() == false); + CHECK(Field.IsString() == true); + CHECK(Field.AsString().empty() == true); + } + } + + SUBCASE("Test Basic Strings") + { + Writer.SetName("String"sv).AddString("Value"sv); + Writer.SetName("String"sv).AddString(L"Value"sv); + + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + for (CbFieldView Field : Fields) + { + CHECK(Field.GetName().compare("String"sv) == 0); + CHECK(Field.HasName() == true); + CHECK(Field.IsString() == true); + CHECK(Field.AsString().compare("Value"sv) == 0); + } + } + + SUBCASE("Long Strings") + { + constexpr int DotCount = 256; + StringBuilder<DotCount + 1> Dots; + for (int Index = 0; Index < DotCount; ++Index) + { + Dots.Append('.'); + } + Writer.AddString(Dots); + Writer.AddString(std::wstring().append(256, L'.')); + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + for (CbFieldView Field : Fields) + { + CHECK((Field.AsString() == std::string_view(Dots))); + } + } + + SUBCASE("Non-ASCII String") + { + wchar_t Value[2] = {0xd83d, 0xde00}; + Writer.AddString("\xf0\x9f\x98\x80"sv); + Writer.AddString(std::wstring_view(Value, ZEN_ARRAY_COUNT(Value))); + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + for (CbFieldView Field : Fields) + { + CHECK((Field.AsString() == "\xf0\x9f\x98\x80"sv)); + } + } +} + +TEST_CASE("usonbuilder.integer") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + auto TestInt32 = [&Writer](int32_t Value) { + Writer.Reset(); + Writer.AddInteger(Value); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + CHECK(Field.AsInt32() == Value); + CHECK(Field.HasError() == false); + }; + + auto TestUInt32 = [&Writer](uint32_t Value) { + Writer.Reset(); + Writer.AddInteger(Value); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + CHECK(Field.AsUInt32() == Value); + CHECK(Field.HasError() == false); + }; + + auto TestInt64 = [&Writer](int64_t Value) { + Writer.Reset(); + Writer.AddInteger(Value); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + CHECK(Field.AsInt64() == Value); + CHECK(Field.HasError() == false); + }; + + auto TestUInt64 = [&Writer](uint64_t Value) { + Writer.Reset(); + Writer.AddInteger(Value); + CbField Field = Writer.Save(); + + CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + CHECK(Field.AsUInt64() == Value); + CHECK(Field.HasError() == false); + }; + + TestUInt32(uint32_t(0x00)); + TestUInt32(uint32_t(0x7f)); + TestUInt32(uint32_t(0x80)); + TestUInt32(uint32_t(0xff)); + TestUInt32(uint32_t(0x0100)); + TestUInt32(uint32_t(0x7fff)); + TestUInt32(uint32_t(0x8000)); + TestUInt32(uint32_t(0xffff)); + TestUInt32(uint32_t(0x0001'0000)); + TestUInt32(uint32_t(0x7fff'ffff)); + TestUInt32(uint32_t(0x8000'0000)); + TestUInt32(uint32_t(0xffff'ffff)); + + TestUInt64(uint64_t(0x0000'0001'0000'0000)); + TestUInt64(uint64_t(0x7fff'ffff'ffff'ffff)); + TestUInt64(uint64_t(0x8000'0000'0000'0000)); + TestUInt64(uint64_t(0xffff'ffff'ffff'ffff)); + + TestInt32(int32_t(0x01)); + TestInt32(int32_t(0x80)); + TestInt32(int32_t(0x81)); + TestInt32(int32_t(0x8000)); + TestInt32(int32_t(0x8001)); + TestInt32(int32_t(0x7fff'ffff)); + TestInt32(int32_t(0x8000'0000)); + TestInt32(int32_t(0x8000'0001)); + + TestInt64(int64_t(0x0000'0001'0000'0000)); + TestInt64(int64_t(0x8000'0000'0000'0000)); + TestInt64(int64_t(0x7fff'ffff'ffff'ffff)); + TestInt64(int64_t(0x8000'0000'0000'0001)); + TestInt64(int64_t(0xffff'ffff'ffff'ffff)); +} + +TEST_CASE("usonbuilder.float") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + SUBCASE("Float32") + { + constexpr float Values[] = { + 0.0f, + 1.0f, + -1.0f, + 3.14159265358979323846f, // PI + 3.402823466e+38f, // FLT_MAX + 1.175494351e-38f // FLT_MIN + }; + + for (float Value : Values) + { + Writer.AddFloat(Value); + } + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + const float* CheckValue = Values; + for (CbFieldView Field : Fields) + { + CHECK(Field.AsFloat() == *CheckValue++); + CHECK(Field.HasError() == false); + } + } + + SUBCASE("Float64") + { + constexpr double Values[] = { + 0.0f, + 1.0f, + -1.0f, + 3.14159265358979323846, // PI + 1.9999998807907104, + 1.9999999403953552, + 3.4028234663852886e38, + 6.8056469327705771e38, + 2.2250738585072014e-308, // DBL_MIN + 1.7976931348623158e+308 // DBL_MAX + }; + + for (double Value : Values) + { + Writer.AddFloat(Value); + } + + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + const double* CheckValue = Values; + for (CbFieldView Field : Fields) + { + CHECK(Field.AsDouble() == *CheckValue++); + CHECK(Field.HasError() == false); + } + } +} + +TEST_CASE("usonbuilder.bool") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + SUBCASE("Bool") + { + Writer.AddBool(true); + Writer.AddBool(false); + + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + CHECK(Fields.AsBool() == true); + CHECK(Fields.HasError() == false); + ++Fields; + CHECK(Fields.AsBool() == false); + CHECK(Fields.HasError() == false); + ++Fields; + CHECK(Fields.HasValue() == false); + } + + SUBCASE("Bool Array/Object Uniformity") + { + Writer.BeginArray(); + Writer.AddBool(false); + Writer.AddBool(false); + Writer.AddBool(false); + Writer.EndArray(); + + Writer.BeginObject(); + Writer.SetName("B1"sv).AddBool(false); + Writer.SetName("B2"sv).AddBool(false); + Writer.SetName("B3"sv).AddBool(false); + Writer.EndObject(); + + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + } +} + +TEST_CASE("usonbuilder.usonattachment") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; +} + +TEST_CASE("usonbuilder.binaryattachment") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; +} + +TEST_CASE("usonbuilder.hash") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; +} + +TEST_CASE("usonbuilder.uuid") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; +} + +TEST_CASE("usonbuilder.datetime") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + const DateTime Values[] = {DateTime(0), DateTime(2020, 5, 13, 15, 10)}; + for (DateTime Value : Values) + { + Writer.AddDateTime(Value); + } + + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + const DateTime* CheckValue = Values; + for (CbFieldView Field : Fields) + { + CHECK(Field.AsDateTime() == *CheckValue++); + CHECK(Field.HasError() == false); + } +} + +TEST_CASE("usonbuilder.timespan") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + const TimeSpan Values[] = {TimeSpan(0), TimeSpan(1, 2, 4, 8)}; + for (TimeSpan Value : Values) + { + Writer.AddTimeSpan(Value); + } + + CbFieldIterator Fields = Writer.Save(); + + CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + + const TimeSpan* CheckValue = Values; + for (CbFieldView Field : Fields) + { + CHECK(Field.AsTimeSpan() == *CheckValue++); + CHECK(Field.HasError() == false); + } +} + +TEST_CASE("usonbuilder.complex") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + SUBCASE("complex") + { + CbObject Object; + + { + Writer.BeginObject(); + + const uint8_t LocalField[] = {uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), 1, 'I', 42}; + Writer.AddField("FieldCopy"sv, CbFieldView(LocalField)); + Writer.AddField("FieldRefCopy"sv, CbField(SharedBuffer::Clone(MakeMemoryView(LocalField)))); + + const uint8_t LocalObject[] = {uint8_t(CbFieldType::Object | CbFieldType::HasFieldName), + 1, + 'O', + 7, + uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), + 1, + 'I', + 42, + uint8_t(CbFieldType::Null | CbFieldType::HasFieldName), + 1, + 'N'}; + Writer.AddObject("ObjectCopy"sv, CbObjectView(LocalObject)); + Writer.AddObject("ObjectRefCopy"sv, CbObject(SharedBuffer::Clone(MakeMemoryView(LocalObject)))); + + const uint8_t LocalArray[] = {uint8_t(CbFieldType::UniformArray | CbFieldType::HasFieldName), + 1, + 'A', + 4, + 2, + uint8_t(CbFieldType::IntegerPositive), + 42, + 21}; + Writer.AddArray("ArrayCopy"sv, CbArrayView(LocalArray)); + Writer.AddArray("ArrayRefCopy"sv, CbArray(SharedBuffer::Clone(MakeMemoryView(LocalArray)))); + + Writer.AddNull("Null"sv); + + Writer.BeginObject("Binary"sv); + { + Writer.AddBinary("Empty"sv, MemoryView()); + Writer.AddBinary("Value"sv, MakeMemoryView("BinaryValue")); + Writer.AddBinary("LargeValue"sv, MakeMemoryView(std::wstring().append(256, L'.'))); + Writer.AddBinary("LargeRefValue"sv, SharedBuffer::Clone(MakeMemoryView(std::wstring().append(256, L'!')))); + } + Writer.EndObject(); + + Writer.BeginObject("Strings"sv); + { + Writer.AddString("AnsiString"sv, "AnsiValue"sv); + Writer.AddString("WideString"sv, std::wstring().append(256, L'.')); + Writer.AddString("EmptyAnsiString"sv, std::string_view()); + Writer.AddString("EmptyWideString"sv, std::wstring_view()); + Writer.AddString("AnsiStringLiteral", "AnsiValue"); + Writer.AddString("WideStringLiteral", L"AnsiValue"); + } + Writer.EndObject(); + + Writer.BeginArray("Integers"sv); + { + Writer.AddInteger(int32_t(-1)); + Writer.AddInteger(int64_t(-1)); + Writer.AddInteger(uint32_t(1)); + Writer.AddInteger(uint64_t(1)); + Writer.AddInteger(std::numeric_limits<int32_t>::min()); + Writer.AddInteger(std::numeric_limits<int32_t>::max()); + Writer.AddInteger(std::numeric_limits<uint32_t>::max()); + Writer.AddInteger(std::numeric_limits<int64_t>::min()); + Writer.AddInteger(std::numeric_limits<int64_t>::max()); + Writer.AddInteger(std::numeric_limits<uint64_t>::max()); + } + Writer.EndArray(); + + Writer.BeginArray("UniformIntegers"sv); + { + Writer.AddInteger(0); + Writer.AddInteger(std::numeric_limits<int32_t>::max()); + Writer.AddInteger(std::numeric_limits<uint32_t>::max()); + Writer.AddInteger(std::numeric_limits<int64_t>::max()); + Writer.AddInteger(std::numeric_limits<uint64_t>::max()); + } + Writer.EndArray(); + + Writer.AddFloat("Float32"sv, 1.0f); + Writer.AddFloat("Float64as32"sv, 2.0); + Writer.AddFloat("Float64"sv, 3.0e100); + + Writer.AddBool("False"sv, false); + Writer.AddBool("True"sv, true); + + Writer.AddCompactBinaryAttachment("CompactBinaryAttachment"sv, IoHash()); + Writer.AddBinaryAttachment("BinaryAttachment"sv, IoHash()); + Writer.AddAttachment("Attachment"sv, CbAttachment()); + + Writer.AddHash("Hash"sv, IoHash()); + Writer.AddUuid("Uuid"sv, Guid()); + + Writer.AddDateTimeTicks("DateTimeZero"sv, 0); + Writer.AddDateTime("DateTime2020"sv, DateTime(2020, 5, 13, 15, 10)); + + Writer.AddTimeSpanTicks("TimeSpanZero"sv, 0); + Writer.AddTimeSpan("TimeSpan"sv, TimeSpan(1, 2, 4, 8)); + + Writer.BeginObject("NestedObjects"sv); + { + Writer.BeginObject("Empty"sv); + Writer.EndObject(); + + Writer.BeginObject("Null"sv); + Writer.AddNull("Null"sv); + Writer.EndObject(); + } + Writer.EndObject(); + + Writer.BeginArray("NestedArrays"sv); + { + Writer.BeginArray(); + Writer.EndArray(); + + Writer.BeginArray(); + Writer.AddNull(); + Writer.AddNull(); + Writer.AddNull(); + Writer.EndArray(); + + Writer.BeginArray(); + Writer.AddBool(false); + Writer.AddBool(false); + Writer.AddBool(false); + Writer.EndArray(); + + Writer.BeginArray(); + Writer.AddBool(true); + Writer.AddBool(true); + Writer.AddBool(true); + Writer.EndArray(); + } + Writer.EndArray(); + + Writer.BeginArray("ArrayOfObjects"sv); + { + Writer.BeginObject(); + Writer.EndObject(); + + Writer.BeginObject(); + Writer.AddNull("Null"sv); + Writer.EndObject(); + } + Writer.EndArray(); + + Writer.BeginArray("LargeArray"sv); + for (int Index = 0; Index < 256; ++Index) + { + Writer.AddInteger(Index - 128); + } + Writer.EndArray(); + + Writer.BeginArray("LargeUniformArray"sv); + for (int Index = 0; Index < 256; ++Index) + { + Writer.AddInteger(Index); + } + Writer.EndArray(); + + Writer.BeginArray("NestedUniformArray"sv); + for (int Index = 0; Index < 16; ++Index) + { + Writer.BeginArray(); + for (int Value = 0; Value < 4; ++Value) + { + Writer.AddInteger(Value); + } + Writer.EndArray(); + } + Writer.EndArray(); + + Writer.EndObject(); + Object = Writer.Save().AsObject(); + } + CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + } +} + +TEST_CASE("usonbuilder.stream") +{ + using namespace std::literals; + + FixedCbWriter<256> Writer; + + SUBCASE("basic") + { + CbObject Object; + { + Writer.BeginObject(); + + const uint8_t LocalField[] = {uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), 1, 'I', 42}; + Writer << "FieldCopy"sv << CbFieldView(LocalField); + + const uint8_t LocalObject[] = {uint8_t(CbFieldType::Object | CbFieldType::HasFieldName), + 1, + 'O', + 7, + uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), + 1, + 'I', + 42, + uint8_t(CbFieldType::Null | CbFieldType::HasFieldName), + 1, + 'N'}; + Writer << "ObjectCopy"sv << CbObjectView(LocalObject); + + const uint8_t LocalArray[] = {uint8_t(CbFieldType::UniformArray | CbFieldType::HasFieldName), + 1, + 'A', + 4, + 2, + uint8_t(CbFieldType::IntegerPositive), + 42, + 21}; + Writer << "ArrayCopy"sv << CbArrayView(LocalArray); + + Writer << "Null"sv << nullptr; + + Writer << "Strings"sv; + Writer.BeginObject(); + Writer << "AnsiString"sv + << "AnsiValue"sv + << "AnsiStringLiteral"sv + << "AnsiValue" + << "WideString"sv << L"WideValue"sv << "WideStringLiteral"sv << L"WideValue"; + Writer.EndObject(); + + Writer << "Integers"sv; + Writer.BeginArray(); + Writer << int32_t(-1) << int64_t(-1) << uint32_t(1) << uint64_t(1); + Writer.EndArray(); + + Writer << "Float32"sv << 1.0f; + Writer << "Float64"sv << 2.0; + + Writer << "False"sv << false << "True"sv << true; + + Writer << "Attachment"sv << CbAttachment(); + + Writer << "Hash"sv << IoHash(); + Writer << "Uuid"sv << Guid(); + + Writer << "DateTime"sv << DateTime(2020, 5, 13, 15, 10); + Writer << "TimeSpan"sv << TimeSpan(1, 2, 4, 8); + + Writer << "LiteralName" << nullptr; + + Writer.EndObject(); + Object = Writer.Save().AsObject(); + } + + CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None); + } +} + +} // namespace zen diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp new file mode 100644 index 000000000..21883f5b1 --- /dev/null +++ b/zencore/compactbinarypackage.cpp @@ -0,0 +1,945 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/compactbinarypackage.h" +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/endian.h> +#include <zencore/stream.h> +#include <zencore/trace.h> + +#include <doctest/doctest.h> + +namespace zen { + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +CbAttachment::CbAttachment(CbFieldIterator InValue, const IoHash* const InHash) +{ + if (InValue) + { + if (!InValue.IsOwned()) + { + InValue = CbFieldIterator::CloneRange(InValue); + } + + CompactBinary = CbFieldViewIterator(InValue); + Buffer = std::move(InValue).GetOuterBuffer(); + } + + if (InHash) + { + Hash = *InHash; + if (CompactBinary) + { + ZEN_ASSERT_SLOW(Hash == CompactBinary.GetRangeHash()); + } + else + { +#if 0 + zenfs_assertSlow(Hash.IsZero(), TEXT("A null or empty field range must use a hash of zero.")); +#endif + } + } + else if (CompactBinary) + { + Hash = CompactBinary.GetRangeHash(); + } +} + +CbAttachment::CbAttachment(SharedBuffer InBuffer, const IoHash* const InHash) : Buffer(std::move(InBuffer)) +{ + Buffer.MakeOwned(); + if (InHash) + { + Hash = *InHash; + if (Buffer.GetSize()) + { + ZEN_ASSERT_SLOW(Hash == IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize())); + } + else + { + ZEN_ASSERT_SLOW(Hash == IoHash::Zero, TEXT("A null or empty buffer must use a hash of zero.")); + } + } + else if (Buffer.GetSize()) + { + Hash = IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize()); + } + else + { + Buffer.Reset(); + } +} + +SharedBuffer +CbAttachment::AsBinaryView() const +{ + if (!CompactBinary) + { + return Buffer; + } + + MemoryView SerializedView; + if (CompactBinary.TryGetSerializedRangeView(SerializedView)) + { + return SerializedView == Buffer.GetView() ? Buffer : SharedBuffer::MakeView(SerializedView, Buffer); + } + + return CbFieldIterator::CloneRange(CompactBinary).GetRangeBuffer(); +} + +CbFieldIterator +CbAttachment::AsCompactBinary() const +{ + return CompactBinary ? CbFieldIterator::MakeRangeView(CompactBinary, Buffer) : CbFieldIterator(); +} + +void +CbAttachment::Load(IoBuffer& InBuffer, BufferAllocator Allocator) +{ + MemoryInStream InStream(InBuffer.Data(), InBuffer.Size()); + BinaryReader Reader(InStream); + + Load(Reader, Allocator); +} + +void +CbAttachment::Load(CbFieldIterator& Fields) +{ + ZEN_ASSERT(Fields.IsBinary()); //, TEXT("Attachments must start with a binary field.")); + const MemoryView View = Fields.AsBinaryView(); + if (View.GetSize() > 0) + { + Buffer = SharedBuffer::MakeView(View, Fields.GetOuterBuffer()); + Buffer.MakeOwned(); + ++Fields; + Hash = Fields.AsAttachment(); + ZEN_ASSERT(!Fields.HasError()); // TEXT("Attachments must be a non-empty binary value with a content hash.")); + if (Fields.IsCompactBinaryAttachment()) + { + CompactBinary = CbFieldViewIterator::MakeRange(Buffer); + } + ++Fields; + } + else + { + ++Fields; + Buffer.Reset(); + CompactBinary.Reset(); + Hash = IoHash::Zero; + } +} + +void +CbAttachment::Load(BinaryReader& Reader, BufferAllocator Allocator) +{ + CbField BufferField = LoadCompactBinary(Reader, Allocator); + ZEN_ASSERT(BufferField.IsBinary(), "Attachments must start with a binary field"); + const MemoryView View = BufferField.AsBinaryView(); + if (View.GetSize() > 0) + { + Buffer = SharedBuffer::MakeView(View, BufferField.GetOuterBuffer()); + Buffer.MakeOwned(); + CompactBinary = CbFieldViewIterator(); + + std::vector<uint8_t> HashBuffer; + CbField HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer { + HashBuffer.resize(Size); + return UniqueBuffer::MakeView(HashBuffer.data(), Size); + }); + Hash = HashField.AsAttachment(); + ZEN_ASSERT(!HashField.HasError(), "Attachments must be a non-empty binary value with a content hash."); + if (HashField.IsCompactBinaryAttachment()) + { + CompactBinary = CbFieldViewIterator::MakeRange(Buffer); + } + } + else + { + Buffer.Reset(); + CompactBinary.Reset(); + Hash = IoHash::Zero; + } +} + +void +CbAttachment::Save(CbWriter& Writer) const +{ + if (CompactBinary) + { + MemoryView SerializedView; + if (CompactBinary.TryGetSerializedRangeView(SerializedView)) + { + Writer.AddBinary(SerializedView); + } + else + { + Writer.AddBinary(AsBinaryView()); + } + Writer.AddCompactBinaryAttachment(Hash); + } + else if (Buffer && Buffer.GetSize()) + { + Writer.AddBinary(Buffer); + Writer.AddBinaryAttachment(Hash); + } + else // Null + { + Writer.AddBinary(MemoryView()); + } +} + +void +CbAttachment::Save(BinaryWriter& Writer) const +{ + CbWriter TempWriter; + Save(TempWriter); + TempWriter.Save(Writer); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void +CbPackage::SetObject(CbObject InObject, const IoHash* InObjectHash, AttachmentResolver* InResolver) +{ + if (InObject.CreateIterator()) + { + Object = InObject.IsOwned() ? std::move(InObject) : CbObject::Clone(InObject); + if (InObjectHash) + { + ObjectHash = *InObjectHash; + ZEN_ASSERT_SLOW(ObjectHash == Object.GetHash()); + } + else + { + ObjectHash = Object.GetHash(); + } + if (InResolver) + { + GatherAttachments(Object.CreateIterator(), *InResolver); + } + } + else + { + Object.Reset(); + ObjectHash = IoHash::Zero; + } +} + +void +CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver) +{ + if (!Attachment.IsNull()) + { + auto It = std::lower_bound(begin(Attachments), end(Attachments), Attachment); + if (It != Attachments.end() && *It == Attachment) + { + CbAttachment& Existing = *It; + if (Attachment.IsCompactBinary() && !Existing.IsCompactBinary()) + { + Existing = CbAttachment(CbFieldIterator::MakeRange(Existing.AsBinaryView())); + } + } + else + { + Attachments.insert(It, Attachment); + } + + if (Attachment.IsCompactBinary() && Resolver) + { + GatherAttachments(Attachment.AsCompactBinary(), *Resolver); + } + } +} + +int32_t +CbPackage::RemoveAttachment(const IoHash& Hash) +{ + return gsl::narrow_cast<int32_t>( + std::erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; })); +} + +bool +CbPackage::Equals(const CbPackage& Package) const +{ + return ObjectHash == Package.ObjectHash && Attachments == Package.Attachments; +} + +const CbAttachment* +CbPackage::FindAttachment(const IoHash& Hash) const +{ + auto It = std::find_if(begin(Attachments), end(Attachments), [&Hash](const CbAttachment& Attachment) -> bool { + return Attachment.GetHash() == Hash; + }); + + if (It == end(Attachments)) + return nullptr; + + return &*It; +} + +void +CbPackage::GatherAttachments(const CbFieldViewIterator& Fields, AttachmentResolver Resolver) +{ + Fields.IterateRangeAttachments([this, &Resolver](CbFieldView Field) { + const IoHash& Hash = Field.AsAttachment(); + + if (SharedBuffer Buffer = Resolver(Hash)) + { + if (Field.IsCompactBinaryAttachment()) + { + AddAttachment(CbAttachment(CbFieldIterator::MakeRange(std::move(Buffer)), Hash), &Resolver); + } + else + { + AddAttachment(CbAttachment(std::move(Buffer), Hash)); + } + } + }); +} + +void +CbPackage::Load(IoBuffer& InBuffer, BufferAllocator Allocator) +{ + MemoryInStream InStream(InBuffer.Data(), InBuffer.Size()); + BinaryReader Reader(InStream); + + Load(Reader, Allocator); +} + +void +CbPackage::Load(CbFieldIterator& Fields) +{ + *this = CbPackage(); + while (Fields) + { + if (Fields.IsNull()) + { + ++Fields; + break; + } + else if (Fields.IsBinary()) + { + CbAttachment Attachment; + Attachment.Load(Fields); + AddAttachment(Attachment); + } + else + { + ZEN_ASSERT(Fields.IsObject(), TEXT("Expected Object, Binary, or Null field when loading a package.")); + Object = Fields.AsObject(); + Object.MakeOwned(); + ++Fields; + if (Object.CreateIterator()) + { + ObjectHash = Fields.AsCompactBinaryAttachment(); + ZEN_ASSERT(!Fields.HasError(), TEXT("Object must be followed by a CompactBinaryReference with the object hash.")); + ++Fields; + } + else + { + Object.Reset(); + } + } + } +} + +void +CbPackage::Load(BinaryReader& Reader, BufferAllocator Allocator) +{ + uint8_t StackBuffer[64]; + const auto StackAllocator = [&Allocator, &StackBuffer](uint64_t Size) -> UniqueBuffer { + if (Size <= sizeof(StackBuffer)) + { + return UniqueBuffer::MakeView(StackBuffer, Size); + } + + return Allocator(Size); + }; + + *this = CbPackage(); + + for (;;) + { + CbField ValueField = LoadCompactBinary(Reader, StackAllocator); + if (ValueField.IsNull()) + { + break; + } + else if (ValueField.IsBinary()) + { + const MemoryView View = ValueField.AsBinaryView(); + if (View.GetSize() > 0) + { + SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer()); + Buffer.MakeOwned(); + CbField HashField = LoadCompactBinary(Reader, StackAllocator); + const IoHash& Hash = HashField.AsAttachment(); + ZEN_ASSERT(!HashField.HasError(), "Attachments must be a non-empty binary value with a content hash."); + if (HashField.IsCompactBinaryAttachment()) + { + AddAttachment(CbAttachment(CbFieldIterator::MakeRange(std::move(Buffer)), Hash)); + } + else + { + AddAttachment(CbAttachment(std::move(Buffer), Hash)); + } + } + } + else + { + ZEN_ASSERT(ValueField.IsObject(), "Expected Object, Binary, or Null field when loading a package"); + Object = ValueField.AsObject(); + Object.MakeOwned(); + if (Object.CreateViewIterator()) + { + CbField HashField = LoadCompactBinary(Reader, StackAllocator); + ObjectHash = HashField.AsCompactBinaryAttachment(); + ZEN_ASSERT(!HashField.HasError(), "Object must be followed by a CompactBinaryAttachment with the object hash."); + } + else + { + Object.Reset(); + } + } + } +} + +void +CbPackage::Save(CbWriter& Writer) const +{ + if (Object.CreateIterator()) + { + Writer.AddObject(Object); + Writer.AddCompactBinaryAttachment(ObjectHash); + } + for (const CbAttachment& Attachment : Attachments) + { + Attachment.Save(Writer); + } + Writer.AddNull(); +} + +void +CbPackage::Save(BinaryWriter& StreamWriter) const +{ + CbWriter Writer; + Save(Writer); + Writer.Save(StreamWriter); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void +usonpackage_forcelink() +{ +} + +TEST_CASE("usonpackage") +{ + using namespace std::literals; + + const auto TestSaveLoadValidate = [&](const char* Test, const CbAttachment& Attachment) { + ZEN_UNUSED(Test); + + CbWriter Writer; + Attachment.Save(Writer); + CbFieldIterator Fields = Writer.Save(); + + MemoryOutStream WriteStream; + BinaryWriter StreamWriter{WriteStream}; + Attachment.Save(StreamWriter); + + CHECK(MakeMemoryView(WriteStream).EqualBytes(Fields.GetRangeBuffer().GetView())); + CHECK(ValidateCompactBinaryRange(MakeMemoryView(WriteStream), CbValidateMode::All) == CbValidateError::None); + CHECK(ValidateCompactBinaryAttachment(MakeMemoryView(WriteStream), CbValidateMode::All) == CbValidateError::None); + + CbAttachment FromFields; + FromFields.Load(Fields); + CHECK(!bool(Fields)); + CHECK(FromFields == Attachment); + + CbAttachment FromArchive; + MemoryInStream InStream(MakeMemoryView(WriteStream)); + BinaryReader Reader(InStream); + FromArchive.Load(Reader); + CHECK(Reader.CurrentOffset() == InStream.Size()); + CHECK(FromArchive == Attachment); + }; + + SUBCASE("Empty Attachment") + { + CbAttachment Attachment; + CHECK(Attachment.IsNull()); + CHECK_FALSE(bool(Attachment)); + CHECK_FALSE(bool(Attachment.AsBinaryView())); + CHECK_FALSE(Attachment.AsCompactBinary().HasValue()); + CHECK_FALSE(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsCompactBinary()); + CHECK(Attachment.GetHash() == IoHash::Zero); + TestSaveLoadValidate("Null", Attachment); + } + + SUBCASE("Binary Attachment") + { + const SharedBuffer Buffer = SharedBuffer::Clone(MakeMemoryView<uint8_t>({0, 1, 2, 3})); + CbAttachment Attachment(Buffer); + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK(Attachment.AsBinaryView() == Buffer); + CHECK_FALSE(Attachment.AsCompactBinary().HasValue()); + CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsCompactBinary()); + CHECK(Attachment.GetHash() == IoHash::HashMemory(Buffer)); + TestSaveLoadValidate("Binary", Attachment); + } + + SUBCASE("Compact Binary Attachment") + { + CbWriter Writer; + Writer << "Name"sv << 42; + CbFieldIterator Fields = Writer.Save(); + CbAttachment Attachment(Fields); + + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK(Attachment.AsBinaryView() == Fields.GetRangeBuffer()); + CHECK(Attachment.AsCompactBinary() == Fields); + CHECK(Attachment.IsBinary()); + CHECK(Attachment.IsCompactBinary()); + CHECK(Attachment.GetHash() == Fields.GetRangeHash()); + TestSaveLoadValidate("CompactBinary", Attachment); + } + + SUBCASE("Binary View") + { + const uint8_t Value[]{0, 1, 2, 3}; + SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value)); + CbAttachment Attachment(Buffer); + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK(Attachment.AsBinaryView().GetView().EqualBytes(Buffer.GetView())); + CHECK_FALSE(Attachment.AsCompactBinary().HasValue()); + CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsCompactBinary()); + CHECK(Attachment.GetHash() == IoHash::HashMemory(Buffer)); + } + + SUBCASE("Compact Binary View") + { + CbWriter Writer; + Writer << "Name"sv << 42; + CbFieldIterator Fields = Writer.Save(); + CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); + CbAttachment Attachment(FieldsView); + + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + + CHECK(Attachment.AsBinaryView() != FieldsView.GetRangeBuffer()); + + CHECK(Attachment.AsCompactBinary().GetRangeView().EqualBytes(Fields.GetRangeView())); + CHECK(Attachment.IsBinary()); + CHECK(Attachment.GetHash() == Fields.GetRangeHash()); + } + + SUBCASE("Binary Load from View") + { + const uint8_t Value[]{0, 1, 2, 3}; + const SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value)); + CbAttachment Attachment(Buffer); + + CbWriter Writer; + Attachment.Save(Writer); + CbFieldIterator Fields = Writer.Save(); + CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); + Attachment.Load(FieldsView); + + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK_FALSE(FieldsView.GetRangeBuffer().GetView().Contains(Attachment.AsBinaryView().GetView())); + CHECK(Attachment.AsBinaryView().GetView().EqualBytes(Buffer.GetView())); + CHECK_FALSE(Attachment.AsCompactBinary().HasValue()); + CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsCompactBinary()); + CHECK(Attachment.GetHash() == IoHash::HashMemory(MakeMemoryView(Value))); + } + + SUBCASE("Compact Binary Load from View") + { + CbWriter ValueWriter; + ValueWriter << "Name"sv << 42; + const CbFieldIterator Value = ValueWriter.Save(); + + CHECK(ValidateCompactBinaryRange(Value.GetRangeView(), CbValidateMode::All) == CbValidateError::None); + CbAttachment Attachment(Value); + + CbWriter Writer; + Attachment.Save(Writer); + CbFieldIterator Fields = Writer.Save(); + CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); + + Attachment.Load(FieldsView); + + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + + CHECK(Attachment.AsBinaryView().GetView().EqualBytes(Value.GetRangeView())); + CHECK_FALSE(FieldsView.GetRangeBuffer().GetView().Contains(Attachment.AsCompactBinary().GetRangeBuffer().GetView())); + CHECK(Attachment.IsBinary()); + CHECK(Attachment.IsCompactBinary()); + + CHECK(Attachment.GetHash() == Value.GetRangeHash()); + } + + SUBCASE("Compact Binary Uniform Sub-View") + { + const SharedBuffer Buffer = SharedBuffer::Clone(MakeMemoryView<uint8_t>({0, 1, 2, 3})); + const CbFieldViewIterator FieldViews = CbFieldViewIterator::MakeRange(Buffer.GetView().RightChop(2), CbFieldType::IntegerPositive); + const CbFieldIterator SavedFields = CbFieldIterator::CloneRange(FieldViews); + CbFieldIterator Fields = CbFieldIterator::MakeRangeView(FieldViews, Buffer); + CbAttachment Attachment(Fields); + const SharedBuffer Binary = Attachment.AsBinaryView(); + CHECK(Attachment.AsCompactBinary() == Fields); + CHECK(Binary.GetSize() == SavedFields.GetRangeSize()); + CHECK(Binary.GetView().EqualBytes(SavedFields.GetRangeView())); + CHECK(Attachment.GetHash() == SavedFields.GetRangeHash()); + TestSaveLoadValidate("CompactBinaryUniformSubView", Attachment); + } + + SUBCASE("Binary Null") + { + const CbAttachment Attachment(SharedBuffer{}); + + CHECK(Attachment.IsNull()); + CHECK_FALSE(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsCompactBinary()); + CHECK(Attachment.GetHash() == IoHash::Zero); + } + + SUBCASE("Binary Empty") + { + const CbAttachment Attachment(SharedBuffer(UniqueBuffer::Alloc(0))); + + CHECK(Attachment.IsNull()); + CHECK_FALSE(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsCompactBinary()); + CHECK(Attachment.GetHash() == IoHash::Zero); + } + + SUBCASE("Compact Binary Empty") + { + const CbAttachment Attachment(CbFieldIterator{}); + + CHECK(Attachment.IsNull()); + CHECK_FALSE(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsCompactBinary()); + CHECK(Attachment.GetHash() == IoHash::Zero); + } +} + +TEST_CASE("usonpackage.serialization") +{ + using namespace std::literals; + + const auto TestSaveLoadValidate = [&](const char* Test, const CbPackage& Package) { + ZEN_UNUSED(Test); + + CbWriter Writer; + Package.Save(Writer); + CbFieldIterator Fields = Writer.Save(); + + MemoryOutStream MemStream; + BinaryWriter WriteAr(MemStream); + Package.Save(WriteAr); + + CHECK(MakeMemoryView(MemStream).EqualBytes(Fields.GetRangeBuffer().GetView())); + CHECK(ValidateCompactBinaryRange(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); + CHECK(ValidateCompactBinaryPackage(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); + + CbPackage FromFields; + FromFields.Load(Fields); + CHECK_FALSE(bool(Fields)); + CHECK(FromFields == Package); + + CbPackage FromArchive; + MemoryInStream ReadMemStream(MakeMemoryView(MemStream)); + BinaryReader ReadAr(ReadMemStream); + FromArchive.Load(ReadAr); + CHECK(ReadAr.CurrentOffset() == ReadMemStream.Size()); + CHECK(FromArchive == Package); + }; + + SUBCASE("Empty") + { + CbPackage Package; + CHECK(Package.IsNull()); + CHECK_FALSE(bool(Package)); + CHECK(Package.GetAttachments().size() == 0); + TestSaveLoadValidate("Empty", Package); + } + + SUBCASE("Object Only") + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Field" << 42; + Writer.EndObject(); + + const CbObject Object = Writer.Save().AsObject(); + CbPackage Package(Object); + CHECK_FALSE(Package.IsNull()); + CHECK(bool(Package)); + CHECK(Package.GetAttachments().size() == 0); + CHECK(Package.GetObject().GetOuterBuffer() == Object.GetOuterBuffer()); + CHECK(Package.GetObject()["Field"].AsInt32() == 42); + CHECK(Package.GetObjectHash() == Package.GetObject().GetHash()); + TestSaveLoadValidate("Object", Package); + } + + // Object View Only + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Field" << 42; + Writer.EndObject(); + + const CbObject Object = Writer.Save().AsObject(); + CbPackage Package(CbObject::MakeView(Object)); + CHECK_FALSE(Package.IsNull()); + CHECK(bool(Package)); + CHECK(Package.GetAttachments().size() == 0); + CHECK(Package.GetObject().GetOuterBuffer() != Object.GetOuterBuffer()); + CHECK(Package.GetObject()["Field"].AsInt32() == 42); + CHECK(Package.GetObjectHash() == Package.GetObject().GetHash()); + TestSaveLoadValidate("Object", Package); + } + + // Attachment Only + { + CbObject Object; + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Field" << 42; + Writer.EndObject(); + Object = Writer.Save().AsObject(); + } + CbField Field = CbField::Clone(Object["Field"]); + + CbPackage Package; + Package.AddAttachment(CbAttachment(CbFieldIterator::MakeSingle(Object.AsField()))); + Package.AddAttachment(CbAttachment(Field.GetBuffer())); + + CHECK_FALSE(Package.IsNull()); + CHECK(bool(Package)); + CHECK(Package.GetAttachments().size() == 2); + CHECK(Package.GetObject().Equals(CbObject())); + CHECK(Package.GetObjectHash() == IoHash()); + TestSaveLoadValidate("Attachments", Package); + + const CbAttachment* const ObjectAttachment = Package.FindAttachment(Object.GetHash()); + REQUIRE(ObjectAttachment); + + const CbAttachment* const FieldAttachment = Package.FindAttachment(Field.GetHash()); + REQUIRE(FieldAttachment); + + CHECK(ObjectAttachment->AsCompactBinary().AsObject().Equals(Object)); + CHECK(FieldAttachment->AsBinaryView() == Field.GetBuffer()); + + Package.AddAttachment(CbAttachment(SharedBuffer::Clone(Object.GetView()))); + Package.AddAttachment(CbAttachment(CbFieldIterator::CloneRange(CbFieldViewIterator::MakeSingle(Field)))); + + CHECK(Package.GetAttachments().size() == 2); + CHECK(Package.FindAttachment(Object.GetHash()) == ObjectAttachment); + CHECK(Package.FindAttachment(Field.GetHash()) == FieldAttachment); + + CHECK(ObjectAttachment->AsCompactBinary().AsObject().Equals(Object)); + CHECK(ObjectAttachment->AsBinaryView() == Object.GetBuffer()); + CHECK(FieldAttachment->AsCompactBinary().Equals(Field)); + CHECK(FieldAttachment->AsBinaryView() == Field.GetBuffer()); + + CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); + } + + // Shared Values + const uint8_t Level4Values[]{0, 1, 2, 3}; + SharedBuffer Level4 = SharedBuffer::MakeView(MakeMemoryView(Level4Values)); + const IoHash Level4Hash = IoHash::HashMemory(Level4); + + CbField Level3; + { + CbWriter Writer; + Writer.SetName("Level4").AddBinaryAttachment(Level4Hash); + Level3 = Writer.Save(); + } + const IoHash Level3Hash = Level3.GetHash(); + + CbArray Level2; + { + CbWriter Writer; + Writer.SetName("Level3"); + Writer.BeginArray(); + Writer.AddCompactBinaryAttachment(Level3Hash); + Writer.EndArray(); + Level2 = Writer.Save().AsArray(); + } + const IoHash Level2Hash = Level2.AsFieldView().GetHash(); + + CbObject Level1; + { + CbWriter Writer; + Writer.BeginObject(); + Writer.SetName("Level2").AddCompactBinaryAttachment(Level2Hash); + Writer.EndObject(); + Level1 = Writer.Save().AsObject(); + } + const IoHash Level1Hash = Level1.AsFieldView().GetHash(); + + const auto Resolver = [&Level2, &Level2Hash, &Level3, &Level3Hash, &Level4, &Level4Hash](const IoHash& Hash) -> SharedBuffer { + return Hash == Level2Hash ? Level2.GetBuffer() + : Hash == Level3Hash ? Level3.GetBuffer() + : Hash == Level4Hash ? Level4 + : SharedBuffer(); + }; + + // Object + Attachments + { + CbPackage Package; + Package.SetObject(Level1, Level1Hash, Resolver); + + CHECK_FALSE(Package.IsNull()); + CHECK(bool(Package)); + CHECK(Package.GetAttachments().size() == 3); + CHECK(Package.GetObject().GetBuffer() == Level1.GetBuffer()); + CHECK(Package.GetObjectHash() == Level1Hash); + TestSaveLoadValidate("Object+Attachments", Package); + + const CbAttachment* const Level2Attachment = Package.FindAttachment(Level2Hash); + const CbAttachment* const Level3Attachment = Package.FindAttachment(Level3Hash); + const CbAttachment* const Level4Attachment = Package.FindAttachment(Level4Hash); + CHECK((Level2Attachment && Level2Attachment->AsCompactBinary().AsArray().Equals(Level2))); + CHECK((Level3Attachment && Level3Attachment->AsCompactBinary().Equals(Level3))); + CHECK((Level4Attachment && Level4Attachment->AsBinaryView() != Level4 && + Level4Attachment->AsBinaryView().GetView().EqualBytes(Level4.GetView()))); + + CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); + + const CbPackage PackageCopy = Package; + CHECK(PackageCopy == Package); + + CHECK(Package.RemoveAttachment(Level1Hash) == 0); + CHECK(Package.RemoveAttachment(Level2Hash) == 1); + CHECK(Package.RemoveAttachment(Level3Hash) == 1); + CHECK(Package.RemoveAttachment(Level4Hash) == 1); + CHECK(Package.RemoveAttachment(Level4Hash) == 0); + CHECK(Package.GetAttachments().size() == 0); + + CHECK(PackageCopy != Package); + Package = PackageCopy; + CHECK(PackageCopy == Package); + Package.SetObject(CbObject()); + CHECK(PackageCopy != Package); + CHECK(Package.GetObjectHash() == IoHash()); + } + + // Out of Order + { + CbWriter Writer; + Writer.AddBinary(Level2.GetBuffer()); + Writer.AddCompactBinaryAttachment(Level2Hash); + Writer.AddBinary(Level4); + Writer.AddBinaryAttachment(Level4Hash); + Writer.AddObject(Level1); + Writer.AddCompactBinaryAttachment(Level1Hash); + Writer.AddBinary(Level3.GetBuffer()); + Writer.AddCompactBinaryAttachment(Level3Hash); + Writer.AddNull(); + + CbFieldIterator Fields = Writer.Save(); + CbPackage FromFields; + FromFields.Load(Fields); + + const CbAttachment* const Level2Attachment = FromFields.FindAttachment(Level2Hash); + REQUIRE(Level2Attachment); + const CbAttachment* const Level3Attachment = FromFields.FindAttachment(Level3Hash); + REQUIRE(Level3Attachment); + const CbAttachment* const Level4Attachment = FromFields.FindAttachment(Level4Hash); + REQUIRE(Level4Attachment); + + CHECK(FromFields.GetObject().Equals(Level1)); + CHECK(FromFields.GetObject().GetOuterBuffer() == Fields.GetOuterBuffer()); + CHECK(FromFields.GetObjectHash() == Level1Hash); + + const MemoryView FieldsOuterBufferView = Fields.GetOuterBuffer().GetView(); + + CHECK(Level2Attachment->AsCompactBinary().AsArray().Equals(Level2)); + CHECK(FieldsOuterBufferView.Contains(Level2Attachment->AsBinaryView().GetView())); + CHECK(Level2Attachment->GetHash() == Level2Hash); + + CHECK(Level3Attachment->AsCompactBinary().Equals(Level3)); + CHECK(FieldsOuterBufferView.Contains(Level3Attachment->AsBinaryView().GetView())); + CHECK(Level3Attachment->GetHash() == Level3Hash); + + CHECK(Level4Attachment->AsBinaryView().GetView().EqualBytes(Level4.GetView())); + CHECK(FieldsOuterBufferView.Contains(Level4Attachment->AsBinaryView().GetView())); + CHECK(Level4Attachment->GetHash() == Level4Hash); + + MemoryOutStream WriteStream; + BinaryWriter WriteAr(WriteStream); + Writer.Save(WriteAr); + CbPackage FromArchive; + MemoryInStream ReadStream(MakeMemoryView(WriteStream)); + BinaryReader ReadAr(ReadStream); + FromArchive.Load(ReadAr); + + Writer.Reset(); + FromArchive.Save(Writer); + CbFieldIterator Saved = Writer.Save(); + CHECK(Saved.AsObject().Equals(Level1)); + ++Saved; + CHECK(Saved.AsCompactBinaryAttachment() == Level1Hash); + ++Saved; + CHECK(Saved.AsBinaryView().EqualBytes(Level2.GetView())); + ++Saved; + CHECK(Saved.AsCompactBinaryAttachment() == Level2Hash); + ++Saved; + CHECK(Saved.AsBinaryView().EqualBytes(Level3.GetView())); + ++Saved; + CHECK(Saved.AsCompactBinaryAttachment() == Level3Hash); + ++Saved; + CHECK(Saved.AsBinaryView().EqualBytes(Level4.GetView())); + ++Saved; + CHECK(Saved.AsBinaryAttachment() == Level4Hash); + ++Saved; + CHECK(Saved.IsNull()); + ++Saved; + CHECK(!Saved); + } + + // Null Attachment + { + const CbAttachment NullAttachment; + CbPackage Package; + Package.AddAttachment(NullAttachment); + CHECK(Package.IsNull()); + CHECK_FALSE(bool(Package)); + CHECK(Package.GetAttachments().size() == 0); + CHECK_FALSE(Package.FindAttachment(NullAttachment)); + } + + // Resolve After Merge + { + bool bResolved = false; + CbPackage Package; + Package.AddAttachment(CbAttachment(Level3.GetBuffer())); + Package.AddAttachment(CbAttachment(CbFieldIterator::MakeSingle(Level3)), [&bResolved](const IoHash& Hash) -> SharedBuffer { + ZEN_UNUSED(Hash); + bResolved = true; + return SharedBuffer(); + }); + CHECK(bResolved); + } +} + +} // namespace zen diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp new file mode 100644 index 000000000..51ed31e95 --- /dev/null +++ b/zencore/compactbinaryvalidation.cpp @@ -0,0 +1,607 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/compactbinaryvalidation.h" + +#include <zencore/compactbinarypackage.h> +#include <zencore/endian.h> +#include <zencore/memory.h> +#include <zencore/string.h> + +#include <algorithm> + +#include <doctest/doctest.h> + +namespace zen { + +namespace CbValidationPrivate { + + template<typename T> + static constexpr inline T ReadUnaligned(const void* const Memory) + { +#if PLATFORM_SUPPORTS_UNALIGNED_LOADS + return *static_cast<const T*>(Memory); +#else + T Value; + memcpy(&Value, Memory, sizeof(Value)); + return Value; +#endif + } + +} // namespace CbValidationPrivate + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Adds the given error(s) to the error mask. + * + * This function exists to make validation errors easier to debug by providing one location to set a breakpoint. + */ +ZEN_NOINLINE static void +AddError(CbValidateError& OutError, const CbValidateError InError) +{ + OutError |= InError; +} + +/** + * Validate and read a field type from the view. + * + * A type argument with the HasFieldType flag indicates that the type will not be read from the view. + */ +static CbFieldType +ValidateCbFieldType(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType Type = CbFieldType::HasFieldType) +{ + ZEN_UNUSED(Mode); + if (CbFieldTypeOps::HasFieldType(Type)) + { + if (View.GetSize() >= 1) + { + Type = *static_cast<const CbFieldType*>(View.GetData()); + View += 1; + if (CbFieldTypeOps::HasFieldType(Type)) + { + AddError(Error, CbValidateError::InvalidType); + } + } + else + { + AddError(Error, CbValidateError::OutOfBounds); + View.Reset(); + return CbFieldType::None; + } + } + + if (CbFieldTypeOps::GetSerializedType(Type) != Type) + { + AddError(Error, CbValidateError::InvalidType); + View.Reset(); + } + + return Type; +} + +/** + * Validate and read an unsigned integer from the view. + * + * Modifies the view to start at the end of the value, and adds error flags if applicable. + */ +static uint64_t +ValidateCbUInt(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + if (View.GetSize() > 0 && View.GetSize() >= MeasureVarUInt(View.GetData())) + { + uint32_t ValueByteCount; + const uint64_t Value = ReadVarUInt(View.GetData(), ValueByteCount); + if (EnumHasAnyFlags(Mode, CbValidateMode::Format) && ValueByteCount > MeasureVarUInt(Value)) + { + AddError(Error, CbValidateError::InvalidInteger); + } + View += ValueByteCount; + return Value; + } + else + { + AddError(Error, CbValidateError::OutOfBounds); + View.Reset(); + return 0; + } +} + +/** + * Validate a 64-bit floating point value from the view. + * + * Modifies the view to start at the end of the value, and adds error flags if applicable. + */ +static void +ValidateCbFloat64(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + if (View.GetSize() >= sizeof(double)) + { + if (EnumHasAnyFlags(Mode, CbValidateMode::Format)) + { + const uint64_t RawValue = FromNetworkOrder(CbValidationPrivate::ReadUnaligned<uint64_t>(View.GetData())); + const double Value = reinterpret_cast<const double&>(RawValue); + if (Value == double(float(Value))) + { + AddError(Error, CbValidateError::InvalidFloat); + } + } + View += sizeof(double); + } + else + { + AddError(Error, CbValidateError::OutOfBounds); + View.Reset(); + } +} + +/** + * Validate and read a string from the view. + * + * Modifies the view to start at the end of the string, and adds error flags if applicable. + */ +static std::string_view +ValidateCbString(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + const uint64_t NameSize = ValidateCbUInt(View, Mode, Error); + if (View.GetSize() >= NameSize) + { + const std::string_view Name(static_cast<const char*>(View.GetData()), static_cast<int32_t>(NameSize)); + View += NameSize; + return Name; + } + else + { + AddError(Error, CbValidateError::OutOfBounds); + View.Reset(); + return std::string_view(); + } +} + +static CbFieldView ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ExternalType); + +/** A type that checks whether all validated fields are of the same type. */ +class CbUniformFieldsValidator +{ +public: + inline explicit CbUniformFieldsValidator(CbFieldType InExternalType) : ExternalType(InExternalType) {} + + inline CbFieldView ValidateField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) + { + const void* const FieldData = View.GetData(); + if (CbFieldView Field = ValidateCbField(View, Mode, Error, ExternalType)) + { + ++FieldCount; + if (CbFieldTypeOps::HasFieldType(ExternalType)) + { + const CbFieldType FieldType = *static_cast<const CbFieldType*>(FieldData); + if (FieldCount == 1) + { + FirstType = FieldType; + } + else if (FieldType != FirstType) + { + bUniform = false; + } + } + return Field; + } + + // It may not safe to check for uniformity if the field was invalid. + bUniform = false; + return CbFieldView(); + } + + inline bool IsUniform() const { return FieldCount > 0 && bUniform; } + +private: + uint32_t FieldCount = 0; + bool bUniform = true; + CbFieldType FirstType = CbFieldType::None; + CbFieldType ExternalType; +}; + +static void +ValidateCbObject(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ObjectType) +{ + const uint64_t Size = ValidateCbUInt(View, Mode, Error); + MemoryView ObjectView = View.Left(Size); + View += Size; + + if (Size > 0) + { + std::vector<std::string_view> Names; + + const bool bUniformObject = CbFieldTypeOps::GetType(ObjectType) == CbFieldType::UniformObject; + const CbFieldType ExternalType = bUniformObject ? ValidateCbFieldType(ObjectView, Mode, Error) : CbFieldType::HasFieldType; + CbUniformFieldsValidator UniformValidator(ExternalType); + do + { + if (CbFieldView Field = UniformValidator.ValidateField(ObjectView, Mode, Error)) + { + if (EnumHasAnyFlags(Mode, CbValidateMode::Names)) + { + if (Field.HasName()) + { + Names.push_back(Field.GetName()); + } + else + { + AddError(Error, CbValidateError::MissingName); + } + } + } + } while (!ObjectView.IsEmpty()); + + if (EnumHasAnyFlags(Mode, CbValidateMode::Names) && Names.size() > 1) + { + std::sort(begin(Names), end(Names), [](std::string_view L, std::string_view R) { return L.compare(R) < 0; }); + + for (const std::string_view *NamesIt = Names.data(), *NamesEnd = NamesIt + Names.size() - 1; NamesIt != NamesEnd; ++NamesIt) + { + if (NamesIt[0] == NamesIt[1]) + { + AddError(Error, CbValidateError::DuplicateName); + break; + } + } + } + + if (!bUniformObject && EnumHasAnyFlags(Mode, CbValidateMode::Format) && UniformValidator.IsUniform()) + { + AddError(Error, CbValidateError::NonUniformObject); + } + } +} + +static void +ValidateCbArray(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ArrayType) +{ + const uint64_t Size = ValidateCbUInt(View, Mode, Error); + MemoryView ArrayView = View.Left(Size); + View += Size; + + const uint64_t Count = ValidateCbUInt(ArrayView, Mode, Error); + const uint64_t FieldsSize = ArrayView.GetSize(); + const bool bUniformArray = CbFieldTypeOps::GetType(ArrayType) == CbFieldType::UniformArray; + const CbFieldType ExternalType = bUniformArray ? ValidateCbFieldType(ArrayView, Mode, Error) : CbFieldType::HasFieldType; + CbUniformFieldsValidator UniformValidator(ExternalType); + + for (uint64_t Index = 0; Index < Count; ++Index) + { + if (CbFieldView Field = UniformValidator.ValidateField(ArrayView, Mode, Error)) + { + if (Field.HasName() && EnumHasAnyFlags(Mode, CbValidateMode::Names)) + { + AddError(Error, CbValidateError::ArrayName); + } + } + } + + if (!bUniformArray && EnumHasAnyFlags(Mode, CbValidateMode::Format) && UniformValidator.IsUniform() && FieldsSize > Count) + { + AddError(Error, CbValidateError::NonUniformArray); + } +} + +static CbFieldView +ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, const CbFieldType ExternalType = CbFieldType::HasFieldType) +{ + const MemoryView FieldView = View; + const CbFieldType Type = ValidateCbFieldType(View, Mode, Error, ExternalType); + const std::string_view Name = CbFieldTypeOps::HasFieldName(Type) ? ValidateCbString(View, Mode, Error) : std::string_view(); + + auto ValidateFixedPayload = [&View, &Error](uint32_t PayloadSize) { + if (View.GetSize() >= PayloadSize) + { + View += PayloadSize; + } + else + { + AddError(Error, CbValidateError::OutOfBounds); + View.Reset(); + } + }; + + if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds | CbValidateError::InvalidType)) + { + return CbFieldView(); + } + + switch (CbFieldType FieldType = CbFieldTypeOps::GetType(Type)) + { + default: + case CbFieldType::None: + AddError(Error, CbValidateError::InvalidType); + View.Reset(); + break; + case CbFieldType::Null: + case CbFieldType::BoolFalse: + case CbFieldType::BoolTrue: + if (FieldView == View) + { + // Reset the view because a zero-sized field can cause infinite field iteration. + AddError(Error, CbValidateError::InvalidType); + View.Reset(); + } + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + ValidateCbObject(View, Mode, Error, FieldType); + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + ValidateCbArray(View, Mode, Error, FieldType); + break; + case CbFieldType::Binary: + { + const uint64_t ValueSize = ValidateCbUInt(View, Mode, Error); + if (View.GetSize() < ValueSize) + { + AddError(Error, CbValidateError::OutOfBounds); + View.Reset(); + } + else + { + View += ValueSize; + } + break; + } + case CbFieldType::String: + ValidateCbString(View, Mode, Error); + break; + case CbFieldType::IntegerPositive: + ValidateCbUInt(View, Mode, Error); + break; + case CbFieldType::IntegerNegative: + ValidateCbUInt(View, Mode, Error); + break; + case CbFieldType::Float32: + ValidateFixedPayload(4); + break; + case CbFieldType::Float64: + ValidateCbFloat64(View, Mode, Error); + break; + case CbFieldType::CompactBinaryAttachment: + case CbFieldType::BinaryAttachment: + case CbFieldType::Hash: + ValidateFixedPayload(20); + break; + case CbFieldType::Uuid: + ValidateFixedPayload(16); + break; + case CbFieldType::DateTime: + case CbFieldType::TimeSpan: + ValidateFixedPayload(8); + break; + case CbFieldType::ObjectId: + ValidateFixedPayload(12); + break; + case CbFieldType::CustomById: + case CbFieldType::CustomByName: + ZEN_NOT_IMPLEMENTED(); // TODO: FIX! + break; + } + + if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds | CbValidateError::InvalidType)) + { + return CbFieldView(); + } + + return CbFieldView(FieldView.GetData(), ExternalType); +} + +static CbFieldView +ValidateCbPackageField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + if (View.IsEmpty()) + { + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + AddError(Error, CbValidateError::InvalidPackageFormat); + } + return CbFieldView(); + } + if (CbFieldView Field = ValidateCbField(View, Mode, Error)) + { + if (Field.HasName() && EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + AddError(Error, CbValidateError::InvalidPackageFormat); + } + return Field; + } + return CbFieldView(); +} + +static IoHash +ValidateCbPackageAttachment(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + const MemoryView ValueView = Value.AsBinaryView(); + if (Value.HasError() && EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + AddError(Error, CbValidateError::InvalidPackageFormat); + } + } + else if (ValueView.GetSize()) + { + if (CbFieldView HashField = ValidateCbPackageField(View, Mode, Error)) + { + const IoHash Hash = HashField.AsAttachment(); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + if (HashField.HasError()) + { + AddError(Error, CbValidateError::InvalidPackageFormat); + } + else if (Hash != IoHash::HashMemory(ValueView.GetData(), ValueView.GetSize())) + { + AddError(Error, CbValidateError::InvalidPackageHash); + } + } + return Hash; + } + } + return IoHash(); +} + +static IoHash +ValidateCbPackageObject(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + CbObjectView Object = Value.AsObjectView(); + if (Value.HasError()) + { + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + AddError(Error, CbValidateError::InvalidPackageFormat); + } + } + else if (CbFieldView HashField = ValidateCbPackageField(View, Mode, Error)) + { + const IoHash Hash = HashField.AsAttachment(); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + if (!Object.CreateViewIterator()) + { + AddError(Error, CbValidateError::NullPackageObject); + } + if (HashField.HasError()) + { + AddError(Error, CbValidateError::InvalidPackageFormat); + } + else if (Hash != Value.GetHash()) + { + AddError(Error, CbValidateError::InvalidPackageHash); + } + } + return Hash; + } + return IoHash(); +} + +CbValidateError +ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type) +{ + CbValidateError Error = CbValidateError::None; + if (EnumHasAnyFlags(Mode, CbValidateMode::All)) + { + ValidateCbField(View, Mode, Error, Type); + if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding)) + { + AddError(Error, CbValidateError::Padding); + } + } + return Error; +} + +CbValidateError +ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode) +{ + CbValidateError Error = CbValidateError::None; + if (EnumHasAnyFlags(Mode, CbValidateMode::All)) + { + while (!View.IsEmpty()) + { + ValidateCbField(View, Mode, Error); + } + } + return Error; +} + +CbValidateError +ValidateCompactBinaryAttachment(MemoryView View, CbValidateMode Mode) +{ + CbValidateError Error = CbValidateError::None; + if (EnumHasAnyFlags(Mode, CbValidateMode::All)) + { + if (CbFieldView Value = ValidateCbPackageField(View, Mode, Error)) + { + ValidateCbPackageAttachment(Value, View, Mode, Error); + } + if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding)) + { + AddError(Error, CbValidateError::Padding); + } + } + return Error; +} + +CbValidateError +ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode) +{ + std::vector<IoHash> Attachments; + CbValidateError Error = CbValidateError::None; + if (EnumHasAnyFlags(Mode, CbValidateMode::All)) + { + uint32_t ObjectCount = 0; + while (CbFieldView Value = ValidateCbPackageField(View, Mode, Error)) + { + if (Value.IsBinary()) + { + const IoHash Hash = ValidateCbPackageAttachment(Value, View, Mode, Error); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + Attachments.push_back(Hash); + if (Value.AsBinaryView().IsEmpty()) + { + AddError(Error, CbValidateError::NullPackageAttachment); + } + } + } + else if (Value.IsObject()) + { + ValidateCbPackageObject(Value, View, Mode, Error); + if (++ObjectCount > 1 && EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + AddError(Error, CbValidateError::MultiplePackageObjects); + } + } + else if (Value.IsNull()) + { + break; + } + else if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + AddError(Error, CbValidateError::InvalidPackageFormat); + } + + if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds)) + { + break; + } + } + + if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding)) + { + AddError(Error, CbValidateError::Padding); + } + + if (Attachments.size() && EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + std::sort(begin(Attachments), end(Attachments)); + for (const IoHash *It = Attachments.data(), *End = It + Attachments.size() - 1; It != End; ++It) + { + if (It[0] == It[1]) + { + AddError(Error, CbValidateError::DuplicateAttachments); + break; + } + } + } + } + return Error; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void +usonvalidation_forcelink() +{ +} + +TEST_CASE("usonvalidation") +{ + SUBCASE("Basic") {} +} + +} // namespace zen diff --git a/zencore/compress.cpp b/zencore/compress.cpp new file mode 100644 index 000000000..0a9c43949 --- /dev/null +++ b/zencore/compress.cpp @@ -0,0 +1,11 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/compress.h> + +#include <doctest/doctest.h> +#include <lz4.h> +#include <functional> + +namespace zen { + +} // namespace zen diff --git a/zencore/except.cpp b/zencore/except.cpp new file mode 100644 index 000000000..b02122f58 --- /dev/null +++ b/zencore/except.cpp @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/except.h> + +namespace zen { + +void +ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] const char* Message) +{ + // TODO + + int ErrValue = hRes; + + throw std::system_error(ErrValue, std::system_category(), Message); +} + +} // namespace zen diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp new file mode 100644 index 000000000..663a626a1 --- /dev/null +++ b/zencore/filesystem.cpp @@ -0,0 +1,592 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/filesystem.h> + +#include <zencore/except.h> +#include <zencore/fmtutils.h> +#include <zencore/iobuffer.h> +#include <zencore/string.h> +#include <zencore/windows.h> + +#include <atlbase.h> +#include <atlfile.h> +#include <winioctl.h> +#include <winnt.h> +#include <filesystem> + +#include <spdlog/spdlog.h> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +using namespace std::literals; + +static bool +DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) +{ + CHandle hDir(CreateFileW(Path, + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + nullptr)); + + if (hDir != INVALID_HANDLE_VALUE) + { + REPARSE_GUID_DATA_BUFFER Rgdb = {}; + Rgdb.ReparseTag = dwReparseTag; + + DWORD dwBytes; + const BOOL bOK = + DeviceIoControl(hDir, FSCTL_DELETE_REPARSE_POINT, &Rgdb, REPARSE_GUID_DATA_BUFFER_HEADER_SIZE, nullptr, 0, &dwBytes, nullptr); + + return bOK == TRUE; + } + + return false; +} + +bool +CreateDirectories(const wchar_t* Dir) +{ + return std::filesystem::create_directories(Dir); +} + +bool +CreateDirectories(const std::filesystem::path& Dir) +{ + return std::filesystem::create_directories(Dir); +} + +// Erase all files and directories in a given directory, leaving an empty directory +// behind + +static bool +WipeDirectory(const wchar_t* DirPath) +{ + ExtendableWideStringBuilder<128> Pattern; + Pattern.Append(DirPath); + Pattern.Append(L"\\*"); + + WIN32_FIND_DATAW FindData; + HANDLE hFind = FindFirstFileW(Pattern.c_str(), &FindData); + + bool AllOk = true; + + if (hFind != nullptr) + { + do + { + bool IsRegular = true; + + if (FindData.cFileName[0] == L'.') + { + if (FindData.cFileName[1] == L'.') + { + if (FindData.cFileName[2] == L'\0') + { + IsRegular = false; + } + } + else if (FindData.cFileName[1] == L'\0') + { + IsRegular = false; + } + } + + if (IsRegular) + { + ExtendableWideStringBuilder<128> Path; + Path.Append(DirPath); + Path.Append(L'\\'); + Path.Append(FindData.cFileName); + + // if (fd.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN) + // deleteReparsePoint(path.c_str(), fd.dwReserved0); + + if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN) + { + DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); + } + + if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) + { + DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); + } + + bool Success = DeleteDirectories(Path.c_str()); + + if (!Success) + { + if (FindData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + { + DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); + } + } + } + else + { + DeleteFileW(Path.c_str()); + } + } + } while (FindNextFileW(hFind, &FindData) == TRUE); + } + + FindClose(hFind); + + return true; +} + +bool +DeleteDirectories(const wchar_t* DirPath) +{ + return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE; +} + +bool +CleanDirectory(const wchar_t* DirPath) +{ + if (std::filesystem::exists(DirPath)) + { + return WipeDirectory(DirPath); + } + else + { + return CreateDirectories(DirPath); + } +} + +bool +DeleteDirectories(const std::filesystem::path& Dir) +{ + return DeleteDirectories(Dir.c_str()); +} + +bool +CleanDirectory(const std::filesystem::path& Dir) +{ + return CleanDirectory(Dir.c_str()); +} + +////////////////////////////////////////////////////////////////////////// + +bool +SupportsBlockRefCounting(std::filesystem::path Path) +{ + ATL::CHandle Handle(CreateFileW(Path.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + nullptr)); + + if (Handle == INVALID_HANDLE_VALUE) + { + Handle.Detach(); + return false; + } + + ULONG FileSystemFlags = 0; + if (!GetVolumeInformationByHandleW(Handle, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0)) + { + return false; + } + + if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING)) + { + return false; + } + + return true; +} + +bool +CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) +{ + ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); + if (FromFile == INVALID_HANDLE_VALUE) + { + FromFile.Detach(); + return false; + } + + ULONG FileSystemFlags; + if (!GetVolumeInformationByHandleW(FromFile, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0)) + { + return false; + } + if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING)) + { + SetLastError(ERROR_NOT_CAPABLE); + return false; + } + + FILE_END_OF_FILE_INFO FileSize; + if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile)) + { + return false; + } + + FILE_BASIC_INFO BasicInfo; + if (!GetFileInformationByHandleEx(FromFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo)) + { + return false; + } + + DWORD dwBytesReturned = 0; + FSCTL_GET_INTEGRITY_INFORMATION_BUFFER GetIntegrityInfoBuffer; + if (!DeviceIoControl(FromFile, + FSCTL_GET_INTEGRITY_INFORMATION, + nullptr, + 0, + &GetIntegrityInfoBuffer, + sizeof GetIntegrityInfoBuffer, + &dwBytesReturned, + nullptr)) + { + return false; + } + + SetFileAttributesW(ToPath.c_str(), FILE_ATTRIBUTE_NORMAL); + + ATL::CHandle TargetFile(CreateFileW(ToPath.c_str(), + GENERIC_READ | GENERIC_WRITE | DELETE, + /* no sharing */ FILE_SHARE_READ, + nullptr, + OPEN_ALWAYS, + 0, + /* hTemplateFile */ FromFile)); + + if (TargetFile == INVALID_HANDLE_VALUE) + { + TargetFile.Detach(); + return false; + } + + // Delete target file when handle is closed (we only reset this if the copy succeeds) + FILE_DISPOSITION_INFO FileDisposition = {TRUE}; + if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)) + { + TargetFile.Close(); + DeleteFileW(ToPath.c_str()); + return false; + } + + // Make file sparse so we don't end up allocating space when we change the file size + if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &dwBytesReturned, nullptr)) + { + return false; + } + + // Copy integrity checking information + FSCTL_SET_INTEGRITY_INFORMATION_BUFFER IntegritySet = {GetIntegrityInfoBuffer.ChecksumAlgorithm, + GetIntegrityInfoBuffer.Reserved, + GetIntegrityInfoBuffer.Flags}; + if (!DeviceIoControl(TargetFile, FSCTL_SET_INTEGRITY_INFORMATION, &IntegritySet, sizeof IntegritySet, nullptr, 0, nullptr, nullptr)) + { + return false; + } + + // Resize file - note that the file is sparse at this point so no additional data will be written + if (!SetFileInformationByHandle(TargetFile, FileEndOfFileInfo, &FileSize, sizeof FileSize)) + { + return false; + } + + constexpr auto RoundToClusterSize = [](LONG64 FileSize, ULONG ClusterSize) -> LONG64 { + return (FileSize + ClusterSize - 1) / ClusterSize * ClusterSize; + }; + static_assert(RoundToClusterSize(5678, 4 * 1024) == 8 * 1024); + + // Loop for cloning file contents. This is necessary as the API has a 32-bit size + // limit for some reason + + const LONG64 SplitThreshold = (1LL << 32) - GetIntegrityInfoBuffer.ClusterSizeInBytes; + + DUPLICATE_EXTENTS_DATA DuplicateExtentsData{.FileHandle = FromFile}; + + for (LONG64 CurrentByteOffset = 0, + RemainingBytes = RoundToClusterSize(FileSize.EndOfFile.QuadPart, GetIntegrityInfoBuffer.ClusterSizeInBytes); + RemainingBytes > 0; + CurrentByteOffset += SplitThreshold, RemainingBytes -= SplitThreshold) + { + DuplicateExtentsData.SourceFileOffset.QuadPart = CurrentByteOffset; + DuplicateExtentsData.TargetFileOffset.QuadPart = CurrentByteOffset; + DuplicateExtentsData.ByteCount.QuadPart = std::min(SplitThreshold, RemainingBytes); + + if (!DeviceIoControl(TargetFile, + FSCTL_DUPLICATE_EXTENTS_TO_FILE, + &DuplicateExtentsData, + sizeof DuplicateExtentsData, + nullptr, + 0, + &dwBytesReturned, + nullptr)) + { + return false; + } + } + + // Make the file not sparse again now that we have populated the contents + if (!(BasicInfo.FileAttributes & FILE_ATTRIBUTE_SPARSE_FILE)) + { + FILE_SET_SPARSE_BUFFER SetSparse = {FALSE}; + + if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, &SetSparse, sizeof SetSparse, nullptr, 0, &dwBytesReturned, nullptr)) + { + return false; + } + } + + // Update timestamps (but don't lie about the creation time) + BasicInfo.CreationTime.QuadPart = 0; + if (!SetFileInformationByHandle(TargetFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo)) + { + return false; + } + + if (!FlushFileBuffers(TargetFile)) + { + return false; + } + + // Finally now everything is done - make sure the file is not deleted on close! + + FileDisposition = {FALSE}; + + const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)); + + return AllOk; +} + +bool +CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) +{ + bool Success = false; + + if (Options.EnableClone) + { + Success = CloneFile(FromPath.native(), ToPath.native()); + + if (Success) + { + return true; + } + } + + if (Options.MustClone) + { + return false; + } + + BOOL CancelFlag = FALSE; + Success = !!::CopyFileExW(FromPath.c_str(), + ToPath.c_str(), + /* lpProgressRoutine */ nullptr, + /* lpData */ nullptr, + &CancelFlag, + /* dwCopyFlags */ 0); + + if (!Success) + { + throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file copy failed"); + } + + return Success; +} + +bool +WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount) +{ + using namespace fmt::literals; + + CAtlFile Outfile; + HRESULT hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); + if (FAILED(hRes)) + { + zen::ThrowIfFailed(hRes, "File open failed for '{}'"_format(Path).c_str()); + } + + // TODO: this could be block-enlightened + + for (size_t i = 0; i < BufferCount; ++i) + { + uint64_t WriteSize = Data[i]->Size(); + const void* DataPtr = Data[i]->Data(); + + while (WriteSize) + { + uint64_t ChunkSize = zen::Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024); + + hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize)); + + if (FAILED(hRes)) + { + zen::ThrowIfFailed(hRes, "File write failed for '{}'"_format(Path).c_str()); + } + + WriteSize -= ChunkSize; + DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize; + } + } + + return true; +} + +FileContents +ReadFile(std::filesystem::path Path) +{ + ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); + if (FromFile == INVALID_HANDLE_VALUE) + { + FromFile.Detach(); + return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())}; + } + + FILE_END_OF_FILE_INFO FileSize; + if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile)) + { + return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())}; + } + + const uint64_t FileSizeBytes = FileSize.EndOfFile.QuadPart; + + FileContents Contents; + + Contents.Data.emplace_back(IoBuffer(IoBuffer::File, FromFile.Detach(), 0, FileSizeBytes)); + + return Contents; +} + +bool +ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc) +{ + ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); + if (FromFile == INVALID_HANDLE_VALUE) + { + FromFile.Detach(); + return false; + } + + std::vector<uint8_t> ReadBuffer(ChunkSize); + + for (;;) + { + DWORD dwBytesRead = 0; + BOOL Success = ::ReadFile(FromFile, ReadBuffer.data(), (DWORD)ReadBuffer.size(), &dwBytesRead, nullptr); + + if (!Success) + { + throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file scan failed"); + } + + if (dwBytesRead == 0) + break; + + ProcessFunc(ReadBuffer.data(), dwBytesRead); + } + + return true; +} + +std::string +ToUtf8(const std::filesystem::path& Path) +{ + return WideToUtf8(Path.native().c_str()); +} + +void +FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) +{ + uint64_t FileInfoBuffer[8 * 1024]; + + FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; + bool Continue = true; + + std::wstring RootDirPath = RootDir.native(); + + CAtlFile RootDirHandle; + HRESULT hRes = RootDirHandle.Create(RootDirPath.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS); + + zen::ThrowIfFailed(hRes, "Failed to open handle to volume root"); + + while (Continue) + { + BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer); + FibClass = FileIdBothDirectoryInfo; // Set up for next iteration + + uint64_t EntryOffset = 0; + + if (!Success) + { + DWORD LastError = GetLastError(); + + if (LastError == ERROR_NO_MORE_FILES) + { + break; + } + + throw std::system_error(std::error_code(LastError, std::system_category()), "file system traversal error"); + } + + for (;;) + { + const FILE_ID_BOTH_DIR_INFO* DirInfo = + reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset); + + std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t)); + + if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + if (FileName == L"."sv || FileName == L".."sv) + { + // Not very interesting + } + else + { + const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName); + + if (ShouldDescend) + { + // Note that this recursion combined with the buffer could + // blow the stack, we should consider a different strategy + + std::filesystem::path FullPath = RootDir / FileName; + + TraverseFileSystem(FullPath, Visitor); + } + } + } + else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE) + { + spdlog::warn("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), RootDir); + } + else + { + std::filesystem::path FullPath = RootDir / FileName; + + Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart); + } + + const uint64_t NextOffset = DirInfo->NextEntryOffset; + + if (NextOffset == 0) + { + break; + } + + EntryOffset += NextOffset; + } + } +} + +} // namespace zen diff --git a/zencore/httpclient.cpp b/zencore/httpclient.cpp new file mode 100644 index 000000000..268483403 --- /dev/null +++ b/zencore/httpclient.cpp @@ -0,0 +1,23 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/httpclient.h> + +#include <spdlog/spdlog.h> + +#include <doctest/doctest.h> + +namespace zen { + +TEST_CASE("httpclient") +{ + using namespace std::literals; + + SUBCASE("client") {} +} + +void +httpclient_forcelink() +{ +} + +} // namespace zen diff --git a/zencore/httpserver.cpp b/zencore/httpserver.cpp new file mode 100644 index 000000000..52389a11b --- /dev/null +++ b/zencore/httpserver.cpp @@ -0,0 +1,1459 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/httpserver.h> + +#define _WINSOCKAPI_ +#include <zencore/windows.h> +#include "iothreadpool.h" + +#include <atlbase.h> +#include <conio.h> +#include <http.h> +#include <new.h> +#include <zencore/compactbinary.h> +#include <zencore/iobuffer.h> +#include <zencore/refcount.h> +#include <zencore/string.h> +#include <zencore/thread.h> +#include <charconv> +#include <span> +#include <string_view> + +#include <spdlog/spdlog.h> + +#include <doctest/doctest.h> + +#pragma comment(lib, "httpapi.lib") + +////////////////////////////////////////////////////////////////////////// + +std::wstring +UTF8_to_wstring(const char* in) +{ + std::wstring out; + unsigned int codepoint; + + while (*in != 0) + { + unsigned char ch = static_cast<unsigned char>(*in); + + if (ch <= 0x7f) + codepoint = ch; + else if (ch <= 0xbf) + codepoint = (codepoint << 6) | (ch & 0x3f); + else if (ch <= 0xdf) + codepoint = ch & 0x1f; + else if (ch <= 0xef) + codepoint = ch & 0x0f; + else + codepoint = ch & 0x07; + + ++in; + + if (((*in & 0xc0) != 0x80) && (codepoint <= 0x10ffff)) + { + if (sizeof(wchar_t) > 2) + { + out.append(1, static_cast<wchar_t>(codepoint)); + } + else if (codepoint > 0xffff) + { + out.append(1, static_cast<wchar_t>(0xd800 + (codepoint >> 10))); + out.append(1, static_cast<wchar_t>(0xdc00 + (codepoint & 0x03ff))); + } + else if (codepoint < 0xd800 || codepoint >= 0xe000) + { + out.append(1, static_cast<wchar_t>(codepoint)); + } + } + } + + return out; +} + +////////////////////////////////////////////////////////////////////////// + +const char* +ReasonStringForHttpResultCode(int HttpCode) +{ + switch (HttpCode) + { + // 1xx Informational + + case 100: + return "Continue"; + case 101: + return "Switching Protocols"; + + // 2xx Success + + case 200: + return "OK"; + case 201: + return "Created"; + case 202: + return "Accepted"; + case 204: + return "No Content"; + case 205: + return "Reset Content"; + case 206: + return "Partial Content"; + + // 3xx Redirection + + case 300: + return "Multiple Choices"; + case 301: + return "Moved Permanently"; + case 302: + return "Found"; + case 303: + return "See Other"; + case 304: + return "Not Modified"; + case 305: + return "Use Proxy"; + case 306: + return "Switch Proxy"; + case 307: + return "Temporary Redirect"; + case 308: + return "Permanent Redirect"; + + // 4xx Client errors + + case 400: + return "Bad Request"; + case 401: + return "Unauthorized"; + case 402: + return "Payment Required"; + case 403: + return "Forbidden"; + case 404: + return "Not Found"; + case 405: + return "Method Not Allowed"; + case 406: + return "Not Acceptable"; + case 407: + return "Proxy Authentication Required"; + case 408: + return "Request Timeout"; + case 409: + return "Conflict"; + case 410: + return "Gone"; + case 411: + return "Length Required"; + case 412: + return "Precondition Failed"; + case 413: + return "Payload Too Large"; + case 414: + return "URI Too Long"; + case 415: + return "Unsupported Media Type"; + case 416: + return "Range Not Satisifiable"; + case 417: + return "Expectation Failed"; + case 418: + return "I'm a teapot"; + case 421: + return "Misdirected Request"; + case 422: + return "Unprocessable Entity"; + case 423: + return "Locked"; + case 424: + return "Failed Dependency"; + case 425: + return "Too Early"; + case 426: + return "Upgrade Required"; + case 428: + return "Precondition Required"; + case 429: + return "Too Many Requests"; + case 431: + return "Request Header Fields Too Large"; + + // 5xx Server errors + + case 500: + return "Internal Server Error"; + case 501: + return "Not Implemented"; + case 502: + return "Bad Gateway"; + case 503: + return "Service Unavailable"; + case 504: + return "Gateway Timeout"; + case 505: + return "HTTP Version Not Supported"; + case 506: + return "Variant Also Negotiates"; + case 507: + return "Insufficient Storage"; + case 508: + return "Loop Detected"; + case 510: + return "Not Extended"; + case 511: + return "Network Authentication Required"; + + default: + return "Unknown Result"; + } +} + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +HttpServerRequest::HttpServerRequest() +{ +} + +HttpServerRequest::~HttpServerRequest() +{ +} + +void +HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, CbObject Data) +{ +#if 0 + struct Visitor : public ICbVisitor + { + virtual void SetName(std::string_view Name) override { OutText << '\'' << Name << "': "; } + virtual void BeginObject() override { OutText << "{ "; } + virtual void EndObject() override { OutText << "}"; } + virtual void BeginArray() override { OutText << "[ "; } + virtual void EndArray() override { OutText << " ]"; } + + virtual void VisitNull() override { OutText << "null"; } + virtual void VisitBinary(SharedBuffer Value) override { ZEN_UNUSED(Value); } + virtual void VisitString(std::string_view Value) override { ZEN_UNUSED(Value); } + virtual void VisitInteger(int64_t Value) override { OutText << Value; } + virtual void VisitInteger(uint64_t Value) override { OutText << Value; } + virtual void VisitFloat(float Value) override { ZEN_UNUSED(Value); } + virtual void VisitDouble(double Value) override { ZEN_UNUSED(Value); } + virtual void VisitBool(bool Value) override { OutText << Value; } + virtual void VisitCbAttachment(const IoHash& Value) override { ZEN_UNUSED(Value); } + virtual void VisitBinaryAttachment(const IoHash& Value) override { ZEN_UNUSED(Value); } + virtual void VisitHash(const IoHash& Value) override { ZEN_UNUSED(Value); } + virtual void VisitUuid(const Guid& Value) override { ZEN_UNUSED(Value); } + virtual void VisitObjectId(const Oid& Value) override { ZEN_UNUSED(Value); } + virtual void VisitDateTime(DateTime Value) override { ZEN_UNUSED(Value); } + virtual void VisitTimeSpan(TimeSpan Value) override { ZEN_UNUSED(Value); } + + ExtendableStringBuilder<256> OutText; + } _; + // Data.CreateRefIterator().VisitFields(_); + return WriteResponse(HttpResponseCode, HttpContentType::kJSON, _.OutText); +#else + SharedBuffer Buf = Data.GetBuffer(); + std::array<IoBuffer, 1> buffers{IoBufferBuilder::MakeCloneFromMemory(Buf.GetData(), Buf.GetSize())}; + return WriteResponse(HttpResponseCode, HttpContentType::kCbObject, buffers); +#endif +} + +void +HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::string_view ResponseString) +{ + return WriteResponse(HttpResponseCode, ContentType, std::u8string_view{(char8_t*)ResponseString.data(), ResponseString.size()}); +} + +void +HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, IoBuffer Blob) +{ + std::array<IoBuffer, 1> buffers{Blob}; + return WriteResponse(HttpResponseCode, ContentType, buffers); +} + +HttpServerRequest::QueryParams +HttpServerRequest::GetQueryParams() +{ + QueryParams Params; + + const std::string_view QStr = QueryString(); + + const char* QueryIt = QStr.data(); + const char* QueryEnd = QueryIt + QStr.size(); + + while (QueryIt != QueryEnd) + { + if (*QueryIt == '&') + { + ++QueryIt; + continue; + } + + const std::string_view Query{QueryIt, QueryEnd}; + + size_t DelimIndex = Query.find('&', 0); + + if (DelimIndex == std::string_view::npos) + { + DelimIndex = Query.size(); + } + + std::string_view ThisQuery{QueryIt, DelimIndex}; + + size_t EqIndex = ThisQuery.find('=', 0); + + if (EqIndex != std::string_view::npos) + { + std::string_view Parm{ThisQuery.data(), EqIndex}; + ThisQuery.remove_prefix(EqIndex + 1); + + Params.KvPairs.emplace_back(Parm, ThisQuery); + } + + QueryIt += DelimIndex; + } + + return std::move(Params); +} + +////////////////////////////////////////////////////////////////////////// +// +// HTTP +// + +class HttpSysServer; +class HttpTransaction; + +class HttpSysRequestHandler +{ +public: + HttpSysRequestHandler(HttpTransaction& InRequest) : m_Request(InRequest) {} + virtual ~HttpSysRequestHandler() = default; + + virtual void IssueRequest() = 0; + virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) = 0; + + HttpTransaction& Transaction() { return m_Request; } + +private: + HttpTransaction& m_Request; // Outermost HTTP transaction object +}; + +/** HTTP transaction + + There will be an instance of this per pending and in-flight HTTP transaction + + */ +class HttpTransaction +{ +public: + HttpTransaction(HttpSysServer& Server) : m_HttpServer(Server), m_HttpHandler(&m_InitialHttpHandler) {} + + virtual ~HttpTransaction() {} + + enum class Status + { + kDone, + kRequestPending + }; + + Status HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred); + + static void __stdcall IoCompletionCallback(PTP_CALLBACK_INSTANCE Instance, + PVOID pContext /* HttpSysServer */, + PVOID pOverlapped, + ULONG IoResult, + ULONG_PTR NumberOfBytesTransferred, + PTP_IO Io) + { + UNREFERENCED_PARAMETER(Io); + UNREFERENCED_PARAMETER(Instance); + UNREFERENCED_PARAMETER(pContext); + + // Note that for a given transaction we may be in this completion function on more + // than one thread at any given moment. This means we need to be careful about what + // happens in here + + HttpTransaction* Transaction = CONTAINING_RECORD(pOverlapped, HttpTransaction, m_HttpOverlapped); + + if (Transaction->HandleCompletion(IoResult, NumberOfBytesTransferred) == HttpTransaction::Status::kDone) + { + delete Transaction; + } + } + + void IssueInitialRequest(); + + PTP_IO Iocp(); + HANDLE RequestQueueHandle(); + inline OVERLAPPED* Overlapped() { return &m_HttpOverlapped; } + inline HttpSysServer& Server() { return m_HttpServer; } + + inline PHTTP_REQUEST HttpRequest() { return m_InitialHttpHandler.HttpRequest(); } + +protected: + OVERLAPPED m_HttpOverlapped{}; + HttpSysServer& m_HttpServer; + HttpSysRequestHandler* m_HttpHandler{nullptr}; + RwLock m_Lock; + +private: + struct InitialRequestHandler : public HttpSysRequestHandler + { + inline PHTTP_REQUEST HttpRequest() { return (PHTTP_REQUEST)m_RequestBuffer; } + inline uint32_t RequestBufferSize() const { return sizeof m_RequestBuffer; } + + InitialRequestHandler(HttpTransaction& InRequest) : HttpSysRequestHandler(InRequest) {} + ~InitialRequestHandler() {} + + virtual void IssueRequest() override; + virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) override; + + PHTTP_REQUEST m_HttpRequestPtr = (HTTP_REQUEST*)(m_RequestBuffer); + UCHAR m_RequestBuffer[16384 + sizeof(HTTP_REQUEST)]; + } m_InitialHttpHandler{*this}; +}; + +////////////////////////////////////////////////////////////////////////// + +class HttpMessageResponseRequest : public HttpSysRequestHandler +{ +public: + HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode); + HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const char* Message); + HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const void* Payload, size_t PayloadSize); + HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, std::span<IoBuffer> Blobs); + ~HttpMessageResponseRequest(); + + virtual void IssueRequest() override; + virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) override; + + void SuppressResponseBody(); + +private: + std::vector<HTTP_DATA_CHUNK> m_HttpDataChunks; + uint64_t m_TotalDataSize = 0; // Sum of all chunk sizes + + uint16_t m_HttpResponseCode = 0; + uint32_t m_NextDataChunkOffset = 0; // This is used for responses where the number of chunks exceed the maximum number for one API call + uint32_t m_RemainingChunkCount = 0; + bool m_IsInitialResponse = true; + + void Initialize(uint16_t ResponseCode, std::span<IoBuffer> Blobs); + + std::vector<IoBuffer> m_DataBuffers; +}; + +HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode) : HttpSysRequestHandler(InRequest) +{ + std::array<IoBuffer, 0> buffers; + + Initialize(ResponseCode, buffers); +} + +HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const char* Message) +: HttpSysRequestHandler(InRequest) +{ + IoBuffer MessageBuffer(IoBuffer::Wrap, Message, strlen(Message)); + std::array<IoBuffer, 1> buffers({MessageBuffer}); + + Initialize(ResponseCode, buffers); +} + +HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, + uint16_t ResponseCode, + const void* Payload, + size_t PayloadSize) +: HttpSysRequestHandler(InRequest) +{ + IoBuffer MessageBuffer(IoBuffer::Wrap, Payload, PayloadSize); + std::array<IoBuffer, 1> buffers({MessageBuffer}); + + Initialize(ResponseCode, buffers); +} + +HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, std::span<IoBuffer> Blobs) +: HttpSysRequestHandler(InRequest) +{ + Initialize(ResponseCode, Blobs); +} + +HttpMessageResponseRequest::~HttpMessageResponseRequest() +{ +} + +void +HttpMessageResponseRequest::Initialize(uint16_t ResponseCode, std::span<IoBuffer> Blobs) +{ + m_HttpResponseCode = ResponseCode; + + const uint32_t ChunkCount = (uint32_t)Blobs.size(); + + m_HttpDataChunks.resize(ChunkCount); + m_DataBuffers.reserve(ChunkCount); + + for (IoBuffer& Buffer : Blobs) + { + m_DataBuffers.emplace_back(std::move(Buffer)).MakeOwned(); + } + + // Initialize the full array up front + + uint64_t LocalDataSize = 0; + + { + PHTTP_DATA_CHUNK ChunkPtr = m_HttpDataChunks.data(); + + for (IoBuffer& Buffer : m_DataBuffers) + { + const ULONG BufferDataSize = (ULONG)Buffer.Size(); + + ZEN_ASSERT(BufferDataSize); + + IoBufferFileReference FileRef; + if (Buffer.GetFileReference(/* out */ FileRef)) + { + ChunkPtr->DataChunkType = HttpDataChunkFromFileHandle; + ChunkPtr->FromFileHandle.FileHandle = FileRef.FileHandle; + ChunkPtr->FromFileHandle.ByteRange.StartingOffset.QuadPart = FileRef.FileChunkOffset; + ChunkPtr->FromFileHandle.ByteRange.Length.QuadPart = BufferDataSize; + } + else + { + ChunkPtr->DataChunkType = HttpDataChunkFromMemory; + ChunkPtr->FromMemory.pBuffer = (void*)Buffer.Data(); + ChunkPtr->FromMemory.BufferLength = BufferDataSize; + } + ++ChunkPtr; + + LocalDataSize += BufferDataSize; + } + } + + m_RemainingChunkCount = ChunkCount; + m_TotalDataSize = LocalDataSize; +} + +void +HttpMessageResponseRequest::SuppressResponseBody() +{ + m_RemainingChunkCount = 0; + m_HttpDataChunks.clear(); + m_DataBuffers.clear(); +} + +HttpSysRequestHandler* +HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) +{ + ZEN_UNUSED(NumberOfBytesTransferred); + ZEN_UNUSED(IoResult); + + if (m_RemainingChunkCount == 0) + return nullptr; // All done + + return this; +} + +void +HttpMessageResponseRequest::IssueRequest() +{ + HttpTransaction& Tx = Transaction(); + HTTP_REQUEST* const HttpReq = Tx.HttpRequest(); + PTP_IO const Iocp = Tx.Iocp(); + + StartThreadpoolIo(Iocp); + + // Split payload into batches to play well with the underlying API + + const int MaxChunksPerCall = 9999; + + const int ThisRequestChunkCount = std::min<int>(m_RemainingChunkCount, MaxChunksPerCall); + const int ThisRequestChunkOffset = m_NextDataChunkOffset; + + m_RemainingChunkCount -= ThisRequestChunkCount; + m_NextDataChunkOffset += ThisRequestChunkCount; + + ULONG SendFlags = 0; + + if (m_RemainingChunkCount) + { + // We need to make more calls to send the full amount of data + SendFlags |= HTTP_SEND_RESPONSE_FLAG_MORE_DATA; + } + + ULONG SendResult = 0; + + if (m_IsInitialResponse) + { + // Populate response structure + + HTTP_RESPONSE HttpResponse = {}; + + HttpResponse.EntityChunkCount = USHORT(ThisRequestChunkCount); + HttpResponse.pEntityChunks = m_HttpDataChunks.data() + ThisRequestChunkOffset; + + // Content-length header + + char ContentLengthString[32]; + _ui64toa_s(m_TotalDataSize, ContentLengthString, sizeof ContentLengthString, 10); + + PHTTP_KNOWN_HEADER ContentLengthHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderContentLength]; + ContentLengthHeader->pRawValue = ContentLengthString; + ContentLengthHeader->RawValueLength = (USHORT)strlen(ContentLengthString); + + // Content-type header + + PHTTP_KNOWN_HEADER ContentTypeHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderContentType]; + + ContentTypeHeader->pRawValue = "application/octet-stream"; /* TODO! We must respect the content type specified */ + ContentTypeHeader->RawValueLength = (USHORT)strlen(ContentTypeHeader->pRawValue); + + HttpResponse.StatusCode = m_HttpResponseCode; + HttpResponse.pReason = ReasonStringForHttpResultCode(m_HttpResponseCode); + HttpResponse.ReasonLength = (USHORT)strlen(HttpResponse.pReason); + + // Cache policy + + HTTP_CACHE_POLICY CachePolicy; + + CachePolicy.Policy = HttpCachePolicyNocache; // HttpCachePolicyUserInvalidates; + CachePolicy.SecondsToLive = 0; + + // Initial response API call + + SendResult = HttpSendHttpResponse(Tx.RequestQueueHandle(), + HttpReq->RequestId, + SendFlags, + &HttpResponse, + &CachePolicy, + NULL, + NULL, + 0, + Tx.Overlapped(), + NULL); + + m_IsInitialResponse = false; + } + else + { + // Subsequent response API calls + + SendResult = HttpSendResponseEntityBody(Tx.RequestQueueHandle(), + HttpReq->RequestId, + SendFlags, + (USHORT)ThisRequestChunkCount, // EntityChunkCount + &m_HttpDataChunks[ThisRequestChunkOffset], // EntityChunks + NULL, // BytesSent + NULL, // Reserved1 + 0, // Reserved2 + Tx.Overlapped(), // Overlapped + NULL // LogData + ); + } + + if ((SendResult != NO_ERROR) // Synchronous completion, but the completion event will still be posted to IOCP + && (SendResult != ERROR_IO_PENDING) // Asynchronous completion + ) + { + // Some error occurred, no completion will be posted + + CancelThreadpoolIo(Iocp); + + spdlog::error("failed to send HTTP response (error: {}) URL: {}", SendResult, HttpReq->pRawUrl); + + throw HttpServerException("Failed to send HTTP response", SendResult); + } +} + +////////////////////////////////////////////////////////////////////////// + +class HttpSysServer +{ + friend class HttpTransaction; + +public: + HttpSysServer(WinIoThreadPool& InThreadPool); + ~HttpSysServer(); + + void Initialize(const wchar_t* UrlPath); + void Run(bool TestMode); + + void RequestExit() { m_ShutdownEvent.Set(); } + + void StartServer(); + void StopServer(); + + void OnHandlingRequest(); + void IssueNewRequestMaybe(); + + inline bool IsOk() const { return m_IsOk; } + + void AddEndpoint(const char* Endpoint, HttpService& Service); + void RemoveEndpoint(const char* Endpoint, HttpService& Service); + +private: + bool m_IsOk = false; + bool m_IsHttpInitialized = false; + WinIoThreadPool& m_ThreadPool; + + std::wstring m_BaseUri; // http://*:nnnn/ + HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; + HTTP_URL_GROUP_ID m_HttpUrlGroupId = 0; + HANDLE m_RequestQueueHandle = 0; + std::atomic_int32_t m_PendingRequests{0}; + int32_t m_MinPendingRequests = 4; + int32_t m_MaxPendingRequests = 32; + Event m_ShutdownEvent; +}; + +HttpSysServer::HttpSysServer(WinIoThreadPool& InThreadPool) : m_ThreadPool(InThreadPool) +{ + ULONG Result = HttpInitialize(HTTPAPI_VERSION_2, HTTP_INITIALIZE_SERVER, nullptr); + + if (Result != NO_ERROR) + { + return; + } + + m_IsHttpInitialized = true; + m_IsOk = true; +} + +HttpSysServer::~HttpSysServer() +{ + if (m_IsHttpInitialized) + { + HttpTerminate(HTTP_INITIALIZE_SERVER, nullptr); + } +} + +void +HttpSysServer::Initialize(const wchar_t* UrlPath) +{ + // check(bIsOk); + + ULONG Result = HttpCreateServerSession(HTTPAPI_VERSION_2, &m_HttpSessionId, 0); + + if (Result != NO_ERROR) + { + // Flag error + + return; + } + + Result = HttpCreateUrlGroup(m_HttpSessionId, &m_HttpUrlGroupId, 0); + + if (Result != NO_ERROR) + { + // Flag error + + return; + } + + m_BaseUri = UrlPath; + + Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, UrlPath, /* #TODO UrlContext */ HTTP_URL_CONTEXT(0), 0); + + if (Result != NO_ERROR) + { + // Flag error + + return; + } + + HTTP_BINDING_INFO HttpBindingInfo = {{0}, 0}; + + Result = HttpCreateRequestQueue(HTTPAPI_VERSION_2, NULL, NULL, 0, &m_RequestQueueHandle); + + if (Result != NO_ERROR) + { + // Flag error! + + return; + } + + HttpBindingInfo.Flags.Present = 1; + HttpBindingInfo.RequestQueueHandle = m_RequestQueueHandle; + + Result = HttpSetUrlGroupProperty(m_HttpUrlGroupId, HttpServerBindingProperty, &HttpBindingInfo, sizeof(HttpBindingInfo)); + + if (Result != NO_ERROR) + { + // Flag error! + + return; + } + + // Create I/O completion port + + m_ThreadPool.CreateIocp(m_RequestQueueHandle, HttpTransaction::IoCompletionCallback, this); + + // Check result! +} + +void +HttpSysServer::StartServer() +{ + int RequestCount = 32; + + for (int i = 0; i < RequestCount; ++i) + { + IssueNewRequestMaybe(); + } +} + +void +HttpSysServer::Run(bool TestMode) +{ + if (TestMode == false) + { + printf("Zen Server running. Press ESC or Q to quit\n"); + } + + bool KeepRunning = true; + + do + { + int WaitTimeout = -1; + + if (!TestMode) + { + WaitTimeout = 1000; + } + + if (!TestMode && _kbhit() != 0) + { + char c = (char)_getch(); + + if (c == 27 || c == 'Q' || c == 'q') + { + RequestApplicationExit(0); + } + } + + m_ShutdownEvent.Wait(WaitTimeout); + } while (!IsApplicationExitRequested()); +} + +void +HttpSysServer::OnHandlingRequest() +{ + --m_PendingRequests; + + if (m_PendingRequests > m_MinPendingRequests) + { + // We have more than the minimum number of requests pending, just let someone else + // enqueue new requests + return; + } + + IssueNewRequestMaybe(); +} + +void +HttpSysServer::IssueNewRequestMaybe() +{ + if (m_PendingRequests.load(std::memory_order::relaxed) >= m_MaxPendingRequests) + { + return; + } + + std::unique_ptr<HttpTransaction> Request = std::make_unique<HttpTransaction>(*this); + + Request->IssueInitialRequest(); + + // This may end up exceeding the MaxPendingRequests limit, but it's not + // really a problem. I'm doing it this way mostly to avoid dealing with + // exceptions here + ++m_PendingRequests; + + Request.release(); +} + +void +HttpSysServer::StopServer() +{ +} + +void +HttpSysServer::AddEndpoint(const char* UrlPath, HttpService& Service) +{ + if (UrlPath[0] == '/') + { + ++UrlPath; + } + + const std::wstring Path16 = UTF8_to_wstring(UrlPath); + Service.SetUriPrefixLength(Path16.size() + 1 /* leading slash */); + + // Convert to wide string + + std::wstring Url16 = m_BaseUri + Path16; + + ULONG Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, Url16.c_str(), HTTP_URL_CONTEXT(&Service), 0 /* Reserved */); + + if (Result != NO_ERROR) + { + spdlog::error("HttpAddUrlToUrlGroup failed with result {}", Result); + + return; + } +} + +void +HttpSysServer::RemoveEndpoint(const char* UrlPath, HttpService& Service) +{ + ZEN_UNUSED(Service); + + if (UrlPath[0] == '/') + { + ++UrlPath; + } + + const std::wstring Path16 = UTF8_to_wstring(UrlPath); + + // Convert to wide string + + std::wstring Url16 = m_BaseUri + Path16; + + ULONG Result = HttpRemoveUrlFromUrlGroup(m_HttpUrlGroupId, Url16.c_str(), 0); + + if (Result != NO_ERROR) + { + spdlog::error("HttpRemoveUrlFromUrlGroup failed with result {}", Result); + } +} + +////////////////////////////////////////////////////////////////////////// + +class HttpSysServerRequest : public HttpServerRequest +{ +public: + HttpSysServerRequest(HttpTransaction& Tx, HttpService& Service) : m_HttpTx(Tx) + { + PHTTP_REQUEST HttpRequestPtr = Tx.HttpRequest(); + + const int PrefixLength = Service.UriPrefixLength(); + const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(char16_t); + + if (AbsPathLength >= PrefixLength) + { + // We convert the URI immediately because most of the code involved prefers to deal + // with utf8. This has some performance impact which I'd prefer to avoid but for now + // we just have to live with it + + WideToUtf8({(char16_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)}, + m_Uri); + } + else + { + m_Uri.Reset(); + } + + if (auto QueryStringLength = HttpRequestPtr->CookedUrl.QueryStringLength) + { + --QueryStringLength; + + WideToUtf8({(char16_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(char16_t)}, m_QueryString); + } + else + { + m_QueryString.Reset(); + } + + switch (HttpRequestPtr->Verb) + { + case HttpVerbOPTIONS: + m_Verb = HttpVerb::kOptions; + break; + + case HttpVerbGET: + m_Verb = HttpVerb::kGet; + break; + + case HttpVerbHEAD: + m_Verb = HttpVerb::kHead; + break; + + case HttpVerbPOST: + m_Verb = HttpVerb::kPost; + break; + + case HttpVerbPUT: + m_Verb = HttpVerb::kPut; + break; + + case HttpVerbDELETE: + m_Verb = HttpVerb::kDelete; + break; + + case HttpVerbCOPY: + m_Verb = HttpVerb::kCopy; + break; + + default: + // TODO: invalid request? + m_Verb = (HttpVerb)0; + break; + } + + auto& clh = HttpRequestPtr->Headers.KnownHeaders[HttpHeaderContentLength]; + std::string_view cl(clh.pRawValue, clh.RawValueLength); + + std::from_chars(cl.data(), cl.data() + cl.size(), m_ContentLength); + } + + ~HttpSysServerRequest() {} + + virtual IoBuffer ReadPayload() override + { + // This is presently synchronous for simplicity, but we + // need to implement an asynchronous version also + + HTTP_REQUEST* const HttpReq = m_HttpTx.HttpRequest(); + + IoBuffer buffer(m_ContentLength); + + uint64_t BytesToRead = m_ContentLength; + + uint8_t* ReadPointer = (uint8_t*)buffer.Data(); + + // First deal with any payload which has already been copied + // into our request buffer + + const int EntityChunkCount = HttpReq->EntityChunkCount; + + for (int i = 0; i < EntityChunkCount; ++i) + { + HTTP_DATA_CHUNK& EntityChunk = HttpReq->pEntityChunks[i]; + + ZEN_ASSERT(EntityChunk.DataChunkType == HttpDataChunkFromMemory); + + const uint64_t BufferLength = EntityChunk.FromMemory.BufferLength; + + ZEN_ASSERT(BufferLength <= BytesToRead); + + memcpy(ReadPointer, EntityChunk.FromMemory.pBuffer, BufferLength); + + ReadPointer += BufferLength; + BytesToRead -= BufferLength; + } + + // Call http.sys API to receive the remaining data + + while (BytesToRead) + { + ULONG BytesRead = 0; + + ULONG ApiResult = HttpReceiveRequestEntityBody(m_HttpTx.RequestQueueHandle(), + HttpReq->RequestId, + 0, /* Flags */ + ReadPointer, + (ULONG)BytesToRead, + &BytesRead, + NULL /* Overlapped */ + ); + + if (ApiResult != NO_ERROR && ApiResult != ERROR_HANDLE_EOF) + { + throw HttpServerException("payload read failed", ApiResult); + } + + BytesToRead -= BytesRead; + ReadPointer += BytesRead; + } + + return buffer; + } + + virtual void WriteResponse(HttpResponse HttpResponseCode) override + { + ZEN_ASSERT(m_IsHandled == false); + + m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode); + + if (m_SuppressBody) + { + m_Response->SuppressResponseBody(); + } + + m_IsHandled = true; + } + + virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) override + { + ZEN_ASSERT(m_IsHandled == false); + ZEN_UNUSED(ContentType); + + m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode, Blobs); + + if (m_SuppressBody) + { + m_Response->SuppressResponseBody(); + } + + m_IsHandled = true; + } + + virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) override + { + ZEN_ASSERT(m_IsHandled == false); + ZEN_UNUSED(ContentType); + + m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode, ResponseString.data(), ResponseString.size()); + + if (m_SuppressBody) + { + m_Response->SuppressResponseBody(); + } + + m_IsHandled = true; + } + + HttpTransaction& m_HttpTx; + HttpMessageResponseRequest* m_Response = nullptr; +}; + +////////////////////////////////////////////////////////////////////////// + +PTP_IO +HttpTransaction::Iocp() +{ + return m_HttpServer.m_ThreadPool.Iocp(); +} + +HANDLE +HttpTransaction::RequestQueueHandle() +{ + return m_HttpServer.m_RequestQueueHandle; +} + +void +HttpTransaction::IssueInitialRequest() +{ + m_InitialHttpHandler.IssueRequest(); +} + +HttpTransaction::Status +HttpTransaction::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) +{ + // We use this to ensure sequential execution of completion handlers + // for any given transaction. + RwLock::ExclusiveLockScope _(m_Lock); + + bool RequestPending = false; + + if (HttpSysRequestHandler* CurrentHandler = m_HttpHandler) + { + const bool IsInitialRequest = (CurrentHandler == &m_InitialHttpHandler); + + if (IsInitialRequest) + { + // Ensure we have a sufficient number of pending requests outstanding + m_HttpServer.OnHandlingRequest(); + } + + m_HttpHandler = CurrentHandler->HandleCompletion(IoResult, NumberOfBytesTransferred); + + if (m_HttpHandler) + { + try + { + m_HttpHandler->IssueRequest(); + + RequestPending = true; + } + catch (std::exception& Ex) + { + spdlog::error("exception caught from IssueRequest(): {}", Ex.what()); + + // something went wrong, no request is pending + } + } + else + { + if (IsInitialRequest == false) + { + delete CurrentHandler; + } + } + } + + m_HttpServer.IssueNewRequestMaybe(); + + if (RequestPending) + { + return Status::kRequestPending; + } + + return Status::kDone; +} + +////////////////////////////////////////////////////////////////////////// + +void +HttpTransaction::InitialRequestHandler::IssueRequest() +{ + PTP_IO Iocp = Transaction().Iocp(); + + StartThreadpoolIo(Iocp); + + HttpTransaction& Tx = Transaction(); + + HTTP_REQUEST* HttpReq = Tx.HttpRequest(); + + ULONG Result = HttpReceiveHttpRequest(Tx.RequestQueueHandle(), + HTTP_NULL_ID, + HTTP_RECEIVE_REQUEST_FLAG_COPY_BODY, + HttpReq, + RequestBufferSize(), + NULL, + Tx.Overlapped()); + + if (Result != ERROR_IO_PENDING && Result != NO_ERROR) + { + CancelThreadpoolIo(Iocp); + + if (Result == ERROR_MORE_DATA) + { + // ProcessReceiveAndPostResponse(pIoRequest, pServerContext->Io, ERROR_MORE_DATA); + } + + // CleanupHttpIoRequest(pIoRequest); + + fprintf(stderr, "HttpReceiveHttpRequest failed, error 0x%lx\n", Result); + + return; + } +} + +HttpSysRequestHandler* +HttpTransaction::InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) +{ + ZEN_UNUSED(IoResult); + ZEN_UNUSED(NumberOfBytesTransferred); + + // Route requests + + try + { + if (HttpService* Service = reinterpret_cast<HttpService*>(m_HttpRequestPtr->UrlContext)) + { + HttpSysServerRequest ThisRequest(Transaction(), *Service); + + Service->HandleRequest(ThisRequest); + + if (!ThisRequest.IsHandled()) + { + return new HttpMessageResponseRequest(Transaction(), 404, "Not found"); + } + + if (ThisRequest.m_Response) + { + return ThisRequest.m_Response; + } + } + + // Unable to route + return new HttpMessageResponseRequest(Transaction(), 404, "Item unknown"); + } + catch (std::exception& ex) + { + // TODO provide more meaningful error output + + return new HttpMessageResponseRequest(Transaction(), 500, ex.what()); + } +} + +////////////////////////////////////////////////////////////////////////// + +struct HttpServer::Impl : public RefCounted +{ + WinIoThreadPool m_ThreadPool; + HttpSysServer m_HttpServer; + + Impl(int ThreadCount) : m_ThreadPool(ThreadCount), m_HttpServer(m_ThreadPool) {} + + void Initialize(int BasePort) + { + using namespace std::literals; + + WideStringBuilder<64> BaseUri; + BaseUri << u8"http://*:"sv << int64_t(BasePort) << u8"/"sv; + + m_HttpServer.Initialize(BaseUri.c_str()); + m_HttpServer.StartServer(); + } + + void Run(bool TestMode) { m_HttpServer.Run(TestMode); } + + void RequestExit() { m_HttpServer.RequestExit(); } + + void Cleanup() { m_HttpServer.StopServer(); } + + void AddEndpoint(const char* Endpoint, HttpService& Service) { m_HttpServer.AddEndpoint(Endpoint, Service); } + + void AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler) + { + ZEN_UNUSED(endpoint, handler); + + ZEN_NOT_IMPLEMENTED(); + } +}; + +HttpServer::HttpServer() +{ + m_Impl = new Impl(32); +} + +HttpServer::~HttpServer() +{ + m_Impl->Cleanup(); +} + +void +HttpServer::AddEndpoint(HttpService& Service) +{ + m_Impl->AddEndpoint(Service.BaseUri(), Service); +} + +void +HttpServer::AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler) +{ + m_Impl->AddEndpoint(endpoint, handler); +} + +void +HttpServer::Initialize(int BasePort) +{ + m_Impl->Initialize(BasePort); +} + +void +HttpServer::Run(bool TestMode) +{ + m_Impl->Run(TestMode); +} + +void +HttpServer::RequestExit() +{ + m_Impl->RequestExit(); +} + +////////////////////////////////////////////////////////////////////////// + +void +HttpRequestRouter::AddPattern(const char* Id, const char* Regex) +{ + ZEN_ASSERT(m_PatternMap.find(Id) == m_PatternMap.end()); + + m_PatternMap.insert({Id, Regex}); +} + +void +HttpRequestRouter::RegisterRoute(const char* Regex, HttpRequestRouter::HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs) +{ + // Expand patterns + + ExtendableStringBuilder<128> ExpandedRegex; + + size_t RegexLen = strlen(Regex); + + for (size_t i = 0; i < RegexLen;) + { + bool matched = false; + + if (Regex[i] == '{' && ((i == 0) || (Regex[i - 1] != '\\'))) + { + // Might have a pattern reference - find closing brace + + for (size_t j = i + 1; j < RegexLen; ++j) + { + if (Regex[j] == '}') + { + std::string Pattern(&Regex[i + 1], j - i - 1); + + if (auto it = m_PatternMap.find(Pattern); it != m_PatternMap.end()) + { + ExpandedRegex.Append(it->second.c_str()); + } + else + { + // Default to anything goes (or should this just be an error?) + + ExpandedRegex.Append("(.+?)"); + } + + // skip ahead + i = j + 1; + + matched = true; + + break; + } + } + } + + if (!matched) + { + ExpandedRegex.Append(Regex[i++]); + } + } + + m_Handlers.emplace_back(ExpandedRegex.c_str(), SupportedVerbs, std::move(HandlerFunc), Regex); +} + +bool +HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) +{ + const HttpVerb Verb = Request.RequestVerb(); + + std::string_view Uri = Request.RelativeUri(); + HttpRouterRequest RouterRequest(Request); + + for (const auto& Handler : m_Handlers) + { + if ((Handler.Verbs & Verb) == Verb && regex_match(begin(Uri), end(Uri), RouterRequest.m_Match, Handler.RegEx)) + { + Handler.Handler(RouterRequest); + + return true; // Route matched + } + } + + return false; // No route matched +} + +TEST_CASE("http") +{ + using namespace std::literals; + + SUBCASE("router") + { + HttpRequestRouter r; + r.AddPattern("a", "[[:alpha:]]+"); + r.RegisterRoute( + "{a}", + [&](auto) {}, + HttpVerb::kGet); + + // struct TestHttpServerRequest : public HttpServerRequest + //{ + // TestHttpServerRequest(std::string_view Uri) : m_uri{Uri} {} + //}; + + // TestHttpServerRequest req{}; + // r.HandleRequest(req); + } +} + +void +http_forcelink() +{ +} + +} // namespace zen diff --git a/zencore/include/zencore/atomic.h b/zencore/include/zencore/atomic.h new file mode 100644 index 000000000..457128bd4 --- /dev/null +++ b/zencore/include/zencore/atomic.h @@ -0,0 +1,43 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <intrin.h> +#include <cinttypes> + +namespace zen { + +inline uint32_t +AtomicIncrement(volatile uint32_t& value) +{ + return _InterlockedIncrement((long volatile*)&value); +} +inline uint32_t +AtomicDecrement(volatile uint32_t& value) +{ + return _InterlockedDecrement((long volatile*)&value); +} + +inline uint64_t +AtomicIncrement(volatile uint64_t& value) +{ + return _InterlockedIncrement64((__int64 volatile*)&value); +} +inline uint64_t +AtomicDecrement(volatile uint64_t& value) +{ + return _InterlockedDecrement64((__int64 volatile*)&value); +} + +inline uint32_t +AtomicAdd(volatile uint32_t& value, uint32_t amount) +{ + return _InterlockedExchangeAdd((long volatile*)&value, amount); +} +inline uint64_t +AtomicAdd(volatile uint64_t& value, uint64_t amount) +{ + return _InterlockedExchangeAdd64((__int64 volatile*)&value, amount); +} + +} // namespace zen diff --git a/zencore/include/zencore/blake3.h b/zencore/include/zencore/blake3.h new file mode 100644 index 000000000..1ef921c30 --- /dev/null +++ b/zencore/include/zencore/blake3.h @@ -0,0 +1,57 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <cinttypes> +#include <compare> +#include <cstring> + +namespace zen { + +class StringBuilderBase; + +/** + * BLAKE3 hash - 256 bits + */ +struct BLAKE3 +{ + uint8_t Hash[32]; + + inline auto operator<=>(const BLAKE3& rhs) const = default; + + static BLAKE3 HashMemory(const void* data, size_t byteCount); + static BLAKE3 FromHexString(const char* string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static const int StringLength = 64; + typedef char String_t[StringLength + 1]; + + static BLAKE3 Zero; // Initialized to all zeroes + + struct Hasher + { + size_t operator()(const BLAKE3& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct BLAKE3Stream +{ + BLAKE3Stream(); + + void Reset(); /// Begin streaming hash compute (not needed on freshly constructed instance) + BLAKE3Stream& Append(const void* data, size_t byteCount); /// Append another chunk + BLAKE3 GetHash(); /// Obtain final hash. If you wish to reuse the instance call reset() + +private: + alignas(16) uint8_t m_HashState[2048]; +}; + +void blake3_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h new file mode 100644 index 000000000..c2d276c21 --- /dev/null +++ b/zencore/include/zencore/compactbinary.h @@ -0,0 +1,1335 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/enumflags.h> +#include <zencore/intmath.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/memory.h> +#include <zencore/meta.h> +#include <zencore/sharedbuffer.h> +#include <zencore/uid.h> +#include <zencore/varint.h> + +#include <functional> +#include <memory> +#include <string> +#include <string_view> +#include <type_traits> +#include <vector> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +class CbObjectView; +class CbArrayView; +class BinaryReader; +class BinaryWriter; + +class DateTime +{ +public: + explicit DateTime(uint64_t InTicks) : Ticks(InTicks) {} + inline DateTime(int Year, int Month, int Day, int Hours = 0, int Minutes = 0, int Seconds = 0, int MilliSeconds = 0) + { + Set(Year, Month, Day, Hours, Minutes, Seconds, MilliSeconds); + } + + inline uint64_t GetTicks() const { return Ticks; } + inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; } + inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; } + +private: + void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond); + uint64_t Ticks; +}; + +class TimeSpan +{ +public: + explicit TimeSpan(uint64_t InTicks) : Ticks(InTicks) {} + inline TimeSpan(int Hours, int Minutes, int Seconds) { Set(0, Hours, Minutes, Seconds, 0); } + inline TimeSpan(int Days, int Hours, int Minutes, int Seconds) { Set(Days, Hours, Minutes, Seconds, 0); } + inline TimeSpan(int Days, int Hours, int Minutes, int Seconds, int Nanos) { Set(Days, Hours, Minutes, Seconds, Nanos); } + + inline uint64_t GetTicks() const { return Ticks; } + inline bool operator==(const TimeSpan& Rhs) const { return Ticks == Rhs.Ticks; } + inline auto operator<=>(const TimeSpan& Rhs) const { return Ticks - Rhs.Ticks; } + + /** + * Time span related constants. + */ + + /** The maximum number of ticks that can be represented in FTimespan. */ + static constexpr int64_t MaxTicks = 9223372036854775807; + + /** The minimum number of ticks that can be represented in FTimespan. */ + static constexpr int64_t MinTicks = -9223372036854775807 - 1; + + /** The number of nanoseconds per tick. */ + static constexpr int64_t NanosecondsPerTick = 100; + + /** The number of timespan ticks per day. */ + static constexpr int64_t TicksPerDay = 864000000000; + + /** The number of timespan ticks per hour. */ + static constexpr int64_t TicksPerHour = 36000000000; + + /** The number of timespan ticks per microsecond. */ + static constexpr int64_t TicksPerMicrosecond = 10; + + /** The number of timespan ticks per millisecond. */ + static constexpr int64_t TicksPerMillisecond = 10000; + + /** The number of timespan ticks per minute. */ + static constexpr int64_t TicksPerMinute = 600000000; + + /** The number of timespan ticks per second. */ + static constexpr int64_t TicksPerSecond = 10000000; + + /** The number of timespan ticks per week. */ + static constexpr int64_t TicksPerWeek = 6048000000000; + + /** The number of timespan ticks per year (365 days, not accounting for leap years). */ + static constexpr int64_t TicksPerYear = 365 * TicksPerDay; + +private: + void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano); + + uint64_t Ticks; +}; + +struct Guid +{ + uint32_t A, B, C, D; +}; + +////////////////////////////////////////////////////////////////////////// + +/** + * Field types and flags for CbField. + * + * This is a private type and is only declared here to enable inline use below. + * + * DO NOT CHANGE THE VALUE OF ANY MEMBERS OF THIS ENUM! + * BACKWARD COMPATIBILITY REQUIRES THAT THESE VALUES BE FIXED! + * SERIALIZATION USES HARD-CODED CONSTANTS BASED ON THESE VALUES! + */ +enum class CbFieldType : uint8_t +{ + /** A field type that does not occur in a valid object. */ + None = 0x00, + + /** Null. Payload is empty. */ + Null = 0x01, + + /** + * Object is an array of fields with unique non-empty names. + * + * Payload is a VarUInt byte count for the encoded fields followed by the fields. + */ + Object = 0x02, + /** + * UniformObject is an array of fields with the same field types and unique non-empty names. + * + * Payload is a VarUInt byte count for the encoded fields followed by the fields. + */ + UniformObject = 0x03, + + /** + * Array is an array of fields with no name that may be of different types. + * + * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by the fields. + */ + Array = 0x04, + /** + * UniformArray is an array of fields with no name and with the same field type. + * + * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by field type, + * followed by the fields without their field type. + */ + UniformArray = 0x05, + + /** Binary. Payload is a VarUInt byte count followed by the data. */ + Binary = 0x06, + + /** String in UTF-8. Payload is a VarUInt byte count then an unterminated UTF-8 string. */ + String = 0x07, + + /** + * Non-negative integer with the range of a 64-bit unsigned integer. + * + * Payload is the value encoded as a VarUInt. + */ + IntegerPositive = 0x08, + /** + * Negative integer with the range of a 64-bit signed integer. + * + * Payload is the ones' complement of the value encoded as a VarUInt. + */ + IntegerNegative = 0x09, + + /** Single precision float. Payload is one big endian IEEE 754 binary32 float. */ + Float32 = 0x0a, + /** Double precision float. Payload is one big endian IEEE 754 binary64 float. */ + Float64 = 0x0b, + + /** Boolean false value. Payload is empty. */ + BoolFalse = 0x0c, + /** Boolean true value. Payload is empty. */ + BoolTrue = 0x0d, + + /** + * CompactBinaryAttachment is a reference to a compact binary attachment stored externally. + * + * Payload is a 160-bit hash digest of the referenced compact binary data. + */ + CompactBinaryAttachment = 0x0e, + /** + * BinaryAttachment is a reference to a binary attachment stored externally. + * + * Payload is a 160-bit hash digest of the referenced binary data. + */ + BinaryAttachment = 0x0f, + + /** Hash. Payload is a 160-bit hash digest. */ + Hash = 0x10, + /** UUID/GUID. Payload is a 128-bit UUID as defined by RFC 4122. */ + Uuid = 0x11, + + /** + * Date and time between 0001-01-01 00:00:00.0000000 and 9999-12-31 23:59:59.9999999. + * + * Payload is a big endian int64 count of 100ns ticks since 0001-01-01 00:00:00.0000000. + */ + DateTime = 0x12, + /** + * Difference between two date/time values. + * + * Payload is a big endian int64 count of 100ns ticks in the span, and may be negative. + */ + TimeSpan = 0x13, + + /** + * Object ID + * + * Payload is a 12-byte opaque identifier + */ + ObjectId = 0x14, + + /** + * CustomById identifies the sub-type of its payload by an integer identifier. + * + * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed + * by a VarUInt of the sub-type identifier then the payload of the sub-type. + */ + CustomById = 0x1e, + /** + * CustomByType identifies the sub-type of its payload by a string identifier. + * + * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed + * by a VarUInt byte count of the unterminated sub-type identifier, then the sub-type identifier + * without termination, then the payload of the sub-type. + */ + CustomByName = 0x1f, + + /** Reserved for future use as a flag. Do not add types in this range. */ + Reserved = 0x20, + + /** + * A transient flag which indicates that the object or array containing this field has stored + * the field type before the payload and name. Non-uniform objects and fields will set this. + * + * Note: Since the flag must never be serialized, this bit may be repurposed in the future. + */ + HasFieldType = 0x40, + + /** A persisted flag which indicates that the field has a name stored before the payload. */ + HasFieldName = 0x80, +}; + +ENUM_CLASS_FLAGS(CbFieldType); + +/** Functions that operate on CbFieldType. */ +class CbFieldTypeOps +{ + static constexpr CbFieldType SerializedTypeMask = CbFieldType(0b1011'1111); + static constexpr CbFieldType TypeMask = CbFieldType(0b0011'1111); + static constexpr CbFieldType ObjectMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType ObjectBase = CbFieldType(0b0000'0010); + static constexpr CbFieldType ArrayMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType ArrayBase = CbFieldType(0b0000'0100); + static constexpr CbFieldType IntegerMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType IntegerBase = CbFieldType(0b0000'1000); + static constexpr CbFieldType FloatMask = CbFieldType(0b0011'1100); + static constexpr CbFieldType FloatBase = CbFieldType(0b0000'1000); + static constexpr CbFieldType BoolMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType BoolBase = CbFieldType(0b0000'1100); + static constexpr CbFieldType AttachmentMask = CbFieldType(0b0011'1110); + static constexpr CbFieldType AttachmentBase = CbFieldType(0b0000'1110); + + static void StaticAssertTypeConstants(); + +public: + /** The type with flags removed. */ + static constexpr inline CbFieldType GetType(CbFieldType Type) { return Type & TypeMask; } + /** The type with transient flags removed. */ + static constexpr inline CbFieldType GetSerializedType(CbFieldType Type) { return Type & SerializedTypeMask; } + + static constexpr inline bool HasFieldType(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldType); } + static constexpr inline bool HasFieldName(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldName); } + + static constexpr inline bool IsNone(CbFieldType Type) { return GetType(Type) == CbFieldType::None; } + static constexpr inline bool IsNull(CbFieldType Type) { return GetType(Type) == CbFieldType::Null; } + + static constexpr inline bool IsObject(CbFieldType Type) { return (Type & ObjectMask) == ObjectBase; } + static constexpr inline bool IsArray(CbFieldType Type) { return (Type & ArrayMask) == ArrayBase; } + + static constexpr inline bool IsBinary(CbFieldType Type) { return GetType(Type) == CbFieldType::Binary; } + static constexpr inline bool IsString(CbFieldType Type) { return GetType(Type) == CbFieldType::String; } + + static constexpr inline bool IsInteger(CbFieldType Type) { return (Type & IntegerMask) == IntegerBase; } + /** Whether the field is a float, or integer due to implicit conversion. */ + static constexpr inline bool IsFloat(CbFieldType Type) { return (Type & FloatMask) == FloatBase; } + static constexpr inline bool IsBool(CbFieldType Type) { return (Type & BoolMask) == BoolBase; } + + static constexpr inline bool IsCompactBinaryAttachment(CbFieldType Type) + { + return GetType(Type) == CbFieldType::CompactBinaryAttachment; + } + static constexpr inline bool IsBinaryAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::BinaryAttachment; } + static constexpr inline bool IsAttachment(CbFieldType Type) { return (Type & AttachmentMask) == AttachmentBase; } + + static constexpr inline bool IsHash(CbFieldType Type) + { + switch (GetType(Type)) + { + case CbFieldType::Hash: + case CbFieldType::BinaryAttachment: + case CbFieldType::CompactBinaryAttachment: + return true; + default: + return false; + } + } + + static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; } + static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; } + + static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; } + static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; } + + /** Whether the type is or may contain fields of any attachment type. */ + static constexpr inline bool MayContainAttachments(CbFieldType Type) + { + // The use of !! will suppress V792 from static analysis. Using //-V792 did not work. + return !!IsObject(Type) | !!IsArray(Type) | !!IsAttachment(Type); + } +}; + +/** Errors that can occur when accessing a field. */ +enum class CbFieldError : uint8_t +{ + /** The field is not in an error state. */ + None, + /** The value type does not match the requested type. */ + TypeError, + /** The value is out of range for the requested type. */ + RangeError, +}; + +class ICbVisitor +{ +public: + virtual void SetName(std::string_view Name) = 0; + virtual void BeginObject() = 0; + virtual void EndObject() = 0; + virtual void BeginArray() = 0; + virtual void EndArray() = 0; + virtual void VisitNull() = 0; + virtual void VisitBinary(SharedBuffer Value) = 0; + virtual void VisitString(std::string_view Value) = 0; + virtual void VisitInteger(int64_t Value) = 0; + virtual void VisitInteger(uint64_t Value) = 0; + virtual void VisitFloat(float Value) = 0; + virtual void VisitDouble(double Value) = 0; + virtual void VisitBool(bool value) = 0; + virtual void VisitCbAttachment(const IoHash& Value) = 0; + virtual void VisitBinaryAttachment(const IoHash& Value) = 0; + virtual void VisitHash(const IoHash& Value) = 0; + virtual void VisitUuid(const Guid& Value) = 0; + virtual void VisitObjectId(const Oid& Value) = 0; + virtual void VisitDateTime(DateTime Value) = 0; + virtual void VisitTimeSpan(TimeSpan Value) = 0; +}; + +/** + * An atom of data in the compact binary format. + * + * Accessing the value of a field is always a safe operation, even if accessed as the wrong type. + * An invalid access will return a default value for the requested type, and set an error code on + * the field that can be checked with GetLastError and HasLastError. A valid access will clear an + * error from a previous invalid access. + * + * A field is encoded in one or more bytes, depending on its type and the type of object or array + * that contains it. A field of an object or array which is non-uniform encodes its field type in + * the first byte, and includes the HasFieldName flag for a field in an object. The field name is + * encoded in a variable-length unsigned integer of its size in bytes, for named fields, followed + * by that many bytes of the UTF-8 encoding of the name with no null terminator. The remainder of + * the field is the payload and is described in the field type enum. Every field must be uniquely + * addressable when encoded, which means a zero-byte field is not permitted, and only arises in a + * uniform array of fields with no payload, where the answer is to encode as a non-uniform array. + * + * This type only provides a view into memory and does not perform any memory management itself. + * Use CbFieldRef to hold a reference to the underlying memory when necessary. + */ + +class CbFieldView +{ +public: + CbFieldView() = default; + + ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType); + + /** Returns the name of the field if it has a name, otherwise an empty view. */ + constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); } + + ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView()); + ZENCORE_API CbObjectView AsObjectView(); + ZENCORE_API CbArrayView AsArrayView(); + ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view()); + + ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const; + + /** Access the field as an int8. Returns the provided default on error. */ + inline int8_t AsInt8(int8_t Default = 0) { return AsInteger<int8_t>(Default); } + /** Access the field as an int16. Returns the provided default on error. */ + inline int16_t AsInt16(int16_t Default = 0) { return AsInteger<int16_t>(Default); } + /** Access the field as an int32. Returns the provided default on error. */ + inline int32_t AsInt32(int32_t Default = 0) { return AsInteger<int32_t>(Default); } + /** Access the field as an int64. Returns the provided default on error. */ + inline int64_t AsInt64(int64_t Default = 0) { return AsInteger<int64_t>(Default); } + /** Access the field as a uint8. Returns the provided default on error. */ + inline uint8_t AsUInt8(uint8_t Default = 0) { return AsInteger<uint8_t>(Default); } + /** Access the field as a uint16. Returns the provided default on error. */ + inline uint16_t AsUInt16(uint16_t Default = 0) { return AsInteger<uint16_t>(Default); } + /** Access the field as a uint32. Returns the provided default on error. */ + inline uint32_t AsUInt32(uint32_t Default = 0) { return AsInteger<uint32_t>(Default); } + /** Access the field as a uint64. Returns the provided default on error. */ + inline uint64_t AsUInt64(uint64_t Default = 0) { return AsInteger<uint64_t>(Default); } + + /** Access the field as a float. Returns the provided default on error. */ + ZENCORE_API float AsFloat(float Default = 0.0f); + /** Access the field as a double. Returns the provided default on error. */ + ZENCORE_API double AsDouble(double Default = 0.0); + + /** Access the field as a bool. Returns the provided default on error. */ + ZENCORE_API bool AsBool(bool bDefault = false); + + /** Access the field as a hash referencing a compact binary attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsCompactBinaryAttachment(const IoHash& Default = IoHash()); + /** Access the field as a hash referencing a binary attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsBinaryAttachment(const IoHash& Default = IoHash()); + /** Access the field as a hash referencing an attachment. Returns the provided default on error. */ + ZENCORE_API IoHash AsAttachment(const IoHash& Default = IoHash()); + + /** Access the field as a hash. Returns the provided default on error. */ + ZENCORE_API IoHash AsHash(const IoHash& Default = IoHash()); + + /** Access the field as a UUID. Returns a nil UUID on error. */ + ZENCORE_API Guid AsUuid(); + /** Access the field as a UUID. Returns the provided default on error. */ + ZENCORE_API Guid AsUuid(const Guid& Default); + + /** Access the field as an OID. Returns a nil OID on error. */ + ZENCORE_API Oid AsObjectId(); + /** Access the field as a OID. Returns the provided default on error. */ + ZENCORE_API Oid AsObjectId(const Oid& Default); + + /** Access the field as a date/time tick count. Returns the provided default on error. */ + ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0); + + /** Access the field as a date/time. Returns a date/time at the epoch on error. */ + ZENCORE_API DateTime AsDateTime(); + /** Access the field as a date/time. Returns the provided default on error. */ + ZENCORE_API DateTime AsDateTime(DateTime Default); + + /** Access the field as a timespan tick count. Returns the provided default on error. */ + ZENCORE_API int64_t AsTimeSpanTicks(int64_t Default = 0); + + /** Access the field as a timespan. Returns an empty timespan on error. */ + ZENCORE_API TimeSpan AsTimeSpan(); + /** Access the field as a timespan. Returns the provided default on error. */ + ZENCORE_API TimeSpan AsTimeSpan(TimeSpan Default); + + /** True if the field has a name. */ + constexpr inline bool HasName() const { return CbFieldTypeOps::HasFieldName(Type); } + + constexpr inline bool IsNull() const { return CbFieldTypeOps::IsNull(Type); } + + constexpr inline bool IsObject() const { return CbFieldTypeOps::IsObject(Type); } + constexpr inline bool IsArray() const { return CbFieldTypeOps::IsArray(Type); } + + constexpr inline bool IsBinary() const { return CbFieldTypeOps::IsBinary(Type); } + constexpr inline bool IsString() const { return CbFieldTypeOps::IsString(Type); } + + /** Whether the field is an integer of unspecified range and sign. */ + constexpr inline bool IsInteger() const { return CbFieldTypeOps::IsInteger(Type); } + /** Whether the field is a float, or integer that supports implicit conversion. */ + constexpr inline bool IsFloat() const { return CbFieldTypeOps::IsFloat(Type); } + constexpr inline bool IsBool() const { return CbFieldTypeOps::IsBool(Type); } + + constexpr inline bool IsCompactBinaryAttachment() const { return CbFieldTypeOps::IsCompactBinaryAttachment(Type); } + constexpr inline bool IsBinaryAttachment() const { return CbFieldTypeOps::IsBinaryAttachment(Type); } + constexpr inline bool IsAttachment() const { return CbFieldTypeOps::IsAttachment(Type); } + + constexpr inline bool IsHash() const { return CbFieldTypeOps::IsHash(Type); } + constexpr inline bool IsUuid() const { return CbFieldTypeOps::IsUuid(Type); } + constexpr inline bool IsObjectId() const { return CbFieldTypeOps::IsObjectId(Type); } + + constexpr inline bool IsDateTime() const { return CbFieldTypeOps::IsDateTime(Type); } + constexpr inline bool IsTimeSpan() const { return CbFieldTypeOps::IsTimeSpan(Type); } + + /** Whether the field has a value. */ + constexpr inline explicit operator bool() const { return HasValue(); } + + /** + * Whether the field has a value. + * + * All fields in a valid object or array have a value. A field with no value is returned when + * finding a field by name fails or when accessing an iterator past the end. + */ + constexpr inline bool HasValue() const { return !CbFieldTypeOps::IsNone(Type); }; + + /** Whether the last field access encountered an error. */ + constexpr inline bool HasError() const { return Error != CbFieldError::None; } + + /** The type of error that occurred on the last field access, or None. */ + constexpr inline CbFieldError GetError() const { return Error; } + + /** Returns the size of the field in bytes, including the type and name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the field, including the type and name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& HashStream) const; + + /** Feed the field (including type and name) to the stream function */ + inline void WriteToStream(auto Hash) const + { + const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type); + Hash(&SerializedType, sizeof(SerializedType)); + auto View = GetViewNoType(); + Hash(View.GetData(), View.GetSize()); + } + + /** Copy the field into a buffer of exactly GetSize() bytes, including the type and name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the field into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + /** + * Whether this field is identical to the other field. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be performed with ValidateCompactBinary, except for field order and field name case. + */ + ZENCORE_API bool Equals(const CbFieldView& Other) const; + + /** Returns a view of the field, including the type and name when present. */ + ZENCORE_API MemoryView GetView() const; + + /** + * Try to get a view of the field as it would be serialized, such as by CopyTo. + * + * A serialized view is not available if the field has an externally-provided type. + * Access the serialized form of such fields using CopyTo or FCbFieldRef::Clone. + */ + inline bool TryGetSerializedView(MemoryView& OutView) const + { + if (CbFieldTypeOps::HasFieldType(Type)) + { + OutView = GetView(); + return true; + } + return false; + } + +protected: + /** Returns a view of the name and value payload, which excludes the type. */ + ZENCORE_API MemoryView GetViewNoType() const; + + /** Returns a view of the value payload, which excludes the type and name. */ + inline MemoryView GetPayloadView() const { return MemoryView(Payload, GetPayloadSize()); } + + /** Returns the type of the field including flags. */ + constexpr inline CbFieldType GetType() const { return Type; } + + /** Returns the start of the value payload. */ + constexpr inline const void* GetPayload() const { return Payload; } + + /** Returns the end of the value payload. */ + inline const void* GetPayloadEnd() const { return static_cast<const uint8_t*>(Payload) + GetPayloadSize(); } + + /** Returns the size of the value payload in bytes, which is the field excluding the type and name. */ + ZENCORE_API uint64_t GetPayloadSize() const; + + /** Assign a field from a pointer to its data and an optional externally-provided type. */ + inline void Assign(const void* InData, const CbFieldType InType) + { + static_assert(std::is_trivially_destructible<CbFieldView>::value, + "This optimization requires CbField to be trivially destructible!"); + new (this) CbFieldView(InData, InType); + } + +private: + /** Parameters for converting to an integer. */ + struct IntegerParams + { + /** Whether the output type has a sign bit. */ + uint32_t IsSigned : 1; + /** Bits of magnitude. (7 for int8) */ + uint32_t MagnitudeBits : 31; + }; + + /** Make integer params for the given integer type. */ + template<typename IntType> + static constexpr inline IntegerParams MakeIntegerParams() + { + IntegerParams Params; + Params.IsSigned = IntType(-1) < IntType(0); + Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned; + return Params; + } + + /** + * Access the field as the given integer type. + * + * Returns the provided default if the value cannot be represented in the output type. + */ + template<typename IntType> + inline IntType AsInteger(IntType Default) + { + return IntType(AsInteger(uint64_t(Default), MakeIntegerParams<IntType>())); + } + + ZENCORE_API uint64_t AsInteger(uint64_t Default, IntegerParams Params); + + /** The field type, with the transient HasFieldType flag if the field contains its type. */ + CbFieldType Type = CbFieldType::None; + /** The error (if any) that occurred on the last field access. */ + CbFieldError Error = CbFieldError::None; + /** The number of bytes for the name stored before the payload. */ + uint32_t NameLen = 0; + /** The value payload, which also points to the end of the name. */ + const void* Payload = nullptr; +}; + +template<typename FieldType> +class TCbFieldIterator : public FieldType +{ +public: + /** Construct an empty field range. */ + constexpr TCbFieldIterator() = default; + + inline TCbFieldIterator& operator++() + { + const void* const PayloadEnd = FieldType::GetPayloadEnd(); + const int64_t AtEndMask = int64_t(PayloadEnd == FieldsEnd) - 1; + const CbFieldType NextType = CbFieldType(int64_t(FieldType::GetType()) & AtEndMask); + const void* const NextField = reinterpret_cast<const void*>(int64_t(PayloadEnd) & AtEndMask); + const void* const NextFieldsEnd = reinterpret_cast<const void*>(int64_t(FieldsEnd) & AtEndMask); + + FieldType::Assign(NextField, NextType); + FieldsEnd = NextFieldsEnd; + return *this; + } + + inline TCbFieldIterator operator++(int) + { + TCbFieldIterator It(*this); + ++*this; + return It; + } + + constexpr inline FieldType& operator*() { return *this; } + constexpr inline FieldType* operator->() { return this; } + + /** Reset this to an empty field range. */ + inline void Reset() { *this = TCbFieldIterator(); } + + /** Returns the size of the fields in the range in bytes. */ + ZENCORE_API uint64_t GetRangeSize() const; + + /** Calculate the hash of every field in the range. */ + ZENCORE_API IoHash GetRangeHash() const; + ZENCORE_API void GetRangeHash(IoHashStream& Hash) const; + + using FieldType::Equals; + + template<typename OtherFieldType> + constexpr inline bool Equals(const TCbFieldIterator<OtherFieldType>& Other) const + { + return FieldType::GetPayload() == Other.OtherFieldType::GetPayload() && FieldsEnd == Other.FieldsEnd; + } + + template<typename OtherFieldType> + constexpr inline bool operator==(const TCbFieldIterator<OtherFieldType>& Other) const + { + return Equals(Other); + } + + template<typename OtherFieldType> + constexpr inline bool operator!=(const TCbFieldIterator<OtherFieldType>& Other) const + { + return !Equals(Other); + } + + /** Copy the field range into a buffer of exactly GetRangeSize() bytes. */ + ZENCORE_API void CopyRangeTo(MutableMemoryView Buffer) const; + + /** Invoke the visitor for every attachment in the field range. */ + ZENCORE_API void IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const; + + /** Create a view of every field in the range. */ + inline MemoryView GetRangeView() const { return MemoryView(FieldType::GetView().GetData(), FieldsEnd); } + + /** + * Try to get a view of every field in the range as they would be serialized. + * + * A serialized view is not available if the underlying fields have an externally-provided type. + * Access the serialized form of such ranges using CbFieldRefIterator::CloneRange. + */ + inline bool TryGetSerializedRangeView(MemoryView& OutView) const + { + if (CbFieldTypeOps::HasFieldType(FieldType::GetType())) + { + OutView = GetRangeView(); + return true; + } + return false; + } + +protected: + /** Construct a field range that contains exactly one field. */ + constexpr inline explicit TCbFieldIterator(FieldType InField) : FieldType(std::move(InField)), FieldsEnd(FieldType::GetPayloadEnd()) {} + + /** + * Construct a field range from the first field and a pointer to the end of the last field. + * + * @param InField The first field, or the default field if there are no fields. + * @param InFieldsEnd A pointer to the end of the payload of the last field, or null. + */ + constexpr inline TCbFieldIterator(FieldType&& InField, const void* InFieldsEnd) : FieldType(std::move(InField)), FieldsEnd(InFieldsEnd) + { + } + + /** Returns the end of the last field, or null for an iterator at the end. */ + template<typename OtherFieldType> + static inline const void* GetFieldsEnd(const TCbFieldIterator<OtherFieldType>& It) + { + return It.FieldsEnd; + } + +private: + friend inline TCbFieldIterator begin(const TCbFieldIterator& Iterator) { return Iterator; } + friend inline TCbFieldIterator end(const TCbFieldIterator&) { return TCbFieldIterator(); } + +private: + template<typename OtherType> + friend class TCbFieldIterator; + + friend class CbFieldViewIterator; + + /** Pointer to the first byte past the end of the last field. Set to null at the end. */ + const void* FieldsEnd = nullptr; +}; + +/** + * Iterator for CbField. + * + * @see CbFieldIterator + */ +class CbFieldViewIterator : public TCbFieldIterator<CbFieldView> +{ +public: + constexpr CbFieldViewIterator() = default; + + /** Construct a field range that contains exactly one field. */ + static inline CbFieldViewIterator MakeSingle(const CbFieldView& Field) { return CbFieldViewIterator(Field); } + + /** + * Construct a field range from a buffer containing zero or more valid fields. + * + * @param View A buffer containing zero or more valid fields. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + */ + static inline CbFieldViewIterator MakeRange(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType) + { + return !View.IsEmpty() ? TCbFieldIterator(CbFieldView(View.GetData(), Type), View.GetDataEnd()) : CbFieldViewIterator(); + } + + /** Construct an iterator from another iterator. */ + template<typename OtherFieldType> + inline CbFieldViewIterator(const TCbFieldIterator<OtherFieldType>& It) + : TCbFieldIterator(ImplicitConv<CbFieldView>(It), GetFieldsEnd(It)) + { + } + +private: + using TCbFieldIterator::TCbFieldIterator; +}; + +/** + * Array of CbField that have no names. + * + * Accessing a field of the array requires iteration. Access by index is not provided because the + * cost of accessing an item by index scales linearly with the index. + * + * This type only provides a view into memory and does not perform any memory management itself. + * Use CbArrayRef to hold a reference to the underlying memory when necessary. + */ +class CbArrayView : protected CbFieldView +{ + friend class CbFieldView; + +public: + /** @see CbField::CbField */ + using CbFieldView::CbFieldView; + + /** Construct an array with no fields. */ + ZENCORE_API CbArrayView(); + + /** Returns the number of items in the array. */ + ZENCORE_API uint64_t Num() const; + + /** Create an iterator for the fields of this array. */ + ZENCORE_API CbFieldViewIterator CreateViewIterator() const; + + /** Visit the fields of this array. */ + ZENCORE_API void VisitFields(ICbVisitor& Visitor); + + /** Access the array as an array field. */ + inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } + + /** Construct an array from an array field. No type check is performed! */ + static inline CbArrayView FromFieldView(const CbFieldView& Field) { return CbArrayView(Field); } + + /** Returns the size of the array in bytes if serialized by itself with no name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the array if serialized by itself with no name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& Stream) const; + + /** + * Whether this array is identical to the other array. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be done with the All mode to check these assumptions about the format of the inputs. + */ + ZENCORE_API bool Equals(const CbArrayView& Other) const; + + /** Copy the array into a buffer of exactly GetSize() bytes, with no name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the array into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + ///** Invoke the visitor for every attachment in the array. */ + inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const + { + CreateViewIterator().IterateRangeAttachments(Visitor); + } + + /** Returns a view of the array, including the type and name when present. */ + using CbFieldView::GetView; + +private: + friend inline CbFieldViewIterator begin(const CbArrayView& Array) { return Array.CreateViewIterator(); } + friend inline CbFieldViewIterator end(const CbArrayView&) { return CbFieldViewIterator(); } + + /** Construct an array from an array field. No type check is performed! Use via FromField. */ + inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {} +}; + +class CbObjectView : protected CbFieldView +{ + friend class CbFieldView; + +public: + /** @see CbField::CbField */ + using CbFieldView::CbFieldView; + + /** Construct an object with no fields. */ + ZENCORE_API CbObjectView(); + + /** Create an iterator for the fields of this object. */ + ZENCORE_API CbFieldViewIterator CreateViewIterator() const; + + /** Visit the fields of this object. */ + ZENCORE_API void VisitFields(ICbVisitor& Visitor); + + /** + * Find a field by case-sensitive name comparison. + * + * The cost of this operation scales linearly with the number of fields in the object. Prefer + * to iterate over the fields only once when consuming an object. + * + * @param Name The name of the field. + * @return The matching field if found, otherwise a field with no value. + */ + ZENCORE_API CbFieldView FindView(std::string_view Name) const; + + /** Find a field by case-insensitive name comparison. */ + ZENCORE_API CbFieldView FindViewIgnoreCase(std::string_view Name) const; + + /** Find a field by case-sensitive name comparison. */ + inline CbFieldView operator[](std::string_view Name) const { return FindView(Name); } + + /** Access the object as an object field. */ + inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } + + /** Construct an object from an object field. No type check is performed! */ + static inline CbObjectView FromFieldView(const CbFieldView& Field) { return CbObjectView(Field); } + + /** Returns the size of the object in bytes if serialized by itself with no name. */ + ZENCORE_API uint64_t GetSize() const; + + /** Calculate the hash of the object if serialized by itself with no name. */ + ZENCORE_API IoHash GetHash() const; + + ZENCORE_API void GetHash(IoHashStream& HashStream) const; + + /** + * Whether this object is identical to the other object. + * + * Performs a deep comparison of any contained arrays or objects and their fields. Comparison + * assumes that both fields are valid and are written in the canonical format. Fields must be + * written in the same order in arrays and objects, and name comparison is case sensitive. If + * these assumptions do not hold, this may return false for equivalent inputs. Validation can + * be done with the All mode to check these assumptions about the format of the inputs. + */ + ZENCORE_API bool Equals(const CbObjectView& Other) const; + + /** Copy the object into a buffer of exactly GetSize() bytes, with no name. */ + ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + + /** Copy the field into an archive, including its type and name. */ + ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + + ///** Invoke the visitor for every attachment in the object. */ + inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const + { + CreateViewIterator().IterateRangeAttachments(Visitor); + } + + /** Returns a view of the object, including the type and name when present. */ + using CbFieldView::GetView; + + /** Whether the field has a value. */ + using CbFieldView::operator bool; + +private: + friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); } + friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); } + + /** Construct an object from an object field. No type check is performed! Use via FromField. */ + inline explicit CbObjectView(const CbFieldView& Field) : CbFieldView(Field) {} +}; + +////////////////////////////////////////////////////////////////////////// + +/** A reference to a function that is used to allocate buffers for compact binary data. */ +using BufferAllocator = std::function<UniqueBuffer(uint64_t Size)>; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** A wrapper that holds a reference to the buffer that contains its compact binary value. */ +template<typename BaseType> +class CbBuffer : public BaseType +{ +public: + /** Construct a default value. */ + CbBuffer() = default; + + /** + * Construct a value from a pointer to its data and an optional externally-provided type. + * + * @param ValueBuffer A buffer that exactly contains the value. + * @param Type HasFieldType means that ValueBuffer contains the type. Otherwise, use the given type. + */ + inline explicit CbBuffer(SharedBuffer ValueBuffer, CbFieldType Type = CbFieldType::HasFieldType) + { + if (ValueBuffer) + { + BaseType::operator=(BaseType(ValueBuffer.GetData(), Type)); + ZEN_ASSERT(ValueBuffer.GetView().Contains(BaseType::GetView())); + Buffer = std::move(ValueBuffer); + } + } + + /** Construct a value that holds a reference to the buffer that contains it. */ + inline CbBuffer(const BaseType& Value, SharedBuffer OuterBuffer) : BaseType(Value) + { + if (OuterBuffer) + { + ZEN_ASSERT(OuterBuffer.GetView().Contains(BaseType::GetView())); + Buffer = std::move(OuterBuffer); + } + } + + /** Construct a value that holds a reference to the buffer of the outer that contains it. */ + template<typename OtherBaseType> + inline CbBuffer(const BaseType& Value, CbBuffer<OtherBaseType> OuterRef) : CbBuffer(Value, std::move(OuterRef.Buffer)) + { + } + + /** Reset this to a default value and null buffer. */ + inline void Reset() { *this = CbBuffer(); } + + /** Whether this reference has ownership of the memory in its buffer. */ + inline bool IsOwned() const { return Buffer && Buffer.IsOwned(); } + + /** Clone the value, if necessary, to a buffer that this reference has ownership of. */ + inline void MakeOwned() + { + if (!IsOwned()) + { + UniqueBuffer MutableBuffer = UniqueBuffer::Alloc(BaseType::GetSize()); + BaseType::CopyTo(MutableBuffer); + BaseType::operator=(BaseType(MutableBuffer.GetData())); + Buffer = std::move(MutableBuffer); + } + } + + /** Returns a buffer that exactly contains this value. */ + inline SharedBuffer GetBuffer() const + { + const MemoryView View = BaseType::GetView(); + const SharedBuffer& OuterBuffer = GetOuterBuffer(); + return View == OuterBuffer.GetView() ? OuterBuffer : SharedBuffer::MakeView(View, OuterBuffer); + } + + /** Returns the outer buffer (if any) that contains this value. */ + inline const SharedBuffer& GetOuterBuffer() const& { return Buffer; } + inline SharedBuffer GetOuterBuffer() && { return std::move(Buffer); } + +private: + template<typename OtherType> + friend class CbBuffer; + + SharedBuffer Buffer; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Factory functions for types derived from CbBuffer. + * + * This uses the curiously recurring template pattern to construct the correct type of reference. + * The derived type inherits from CbBufferRef and this type to expose the factory functions. + */ +template<typename RefType, typename BaseType> +class CbBufferFactory +{ +public: + /** Construct a value from an owned clone of its memory. */ + static inline RefType Clone(const void* const Data) { return Clone(BaseType(Data)); } + + /** Construct a value from an owned clone of its memory. */ + static inline RefType Clone(const BaseType& Value) + { + RefType Ref = MakeView(Value); + Ref.MakeOwned(); + return Ref; + } + + /** Construct a value from a read-only view of its memory and its optional outer buffer. */ + static inline RefType MakeView(const void* const Data, SharedBuffer OuterBuffer = SharedBuffer()) + { + return MakeView(BaseType(Data), std::move(OuterBuffer)); + } + + /** Construct a value from a read-only view of its memory and its optional outer buffer. */ + static inline RefType MakeView(const BaseType& Value, SharedBuffer OuterBuffer = SharedBuffer()) + { + return RefType(Value, std::move(OuterBuffer)); + } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class CbArray; +class CbObject; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A field that can hold a reference to the memory that contains it. + * + * @see CbBufferRef + */ +class CbField : public CbBuffer<CbFieldView>, public CbBufferFactory<CbField, CbFieldView> +{ +public: + using CbBuffer::CbBuffer; + + /** Access the field as an object. Defaults to an empty object on error. */ + inline CbObject AsObject() &; + + /** Access the field as an object. Defaults to an empty object on error. */ + inline CbObject AsObject() &&; + + /** Access the field as an array. Defaults to an empty array on error. */ + inline CbArray AsArray() &; + + /** Access the field as an array. Defaults to an empty array on error. */ + inline CbArray AsArray() &&; +}; + +/** + * Iterator for CbFieldRef. + * + * @see CbFieldIterator + */ +class CbFieldIterator : public TCbFieldIterator<CbField> +{ +public: + /** Construct a field range from an owned clone of a range. */ + ZENCORE_API static CbFieldIterator CloneRange(const CbFieldViewIterator& It); + + /** Construct a field range from an owned clone of a range. */ + static inline CbFieldIterator CloneRange(const CbFieldIterator& It) { return CloneRange(CbFieldViewIterator(It)); } + + /** Construct a field range that contains exactly one field. */ + static inline CbFieldIterator MakeSingle(CbField Field) { return CbFieldIterator(std::move(Field)); } + + /** + * Construct a field range from a buffer containing zero or more valid fields. + * + * @param Buffer A buffer containing zero or more valid fields. + * @param Type HasFieldType means that Buffer contains the type. Otherwise, use the given type. + */ + static inline CbFieldIterator MakeRange(SharedBuffer Buffer, CbFieldType Type = CbFieldType::HasFieldType) + { + if (Buffer.GetSize()) + { + const void* const DataEnd = Buffer.GetView().GetDataEnd(); + return CbFieldIterator(CbField(std::move(Buffer), Type), DataEnd); + } + return CbFieldIterator(); + } + + /** Construct a field range from an iterator and its optional outer buffer. */ + static inline CbFieldIterator MakeRangeView(const CbFieldViewIterator& It, SharedBuffer OuterBuffer = SharedBuffer()) + { + return CbFieldIterator(CbField(It, std::move(OuterBuffer)), GetFieldsEnd(It)); + } + + /** Construct an empty field range. */ + constexpr CbFieldIterator() = default; + + /** Clone the range, if necessary, to a buffer that this reference has ownership of. */ + inline void MakeRangeOwned() + { + if (!IsOwned()) + { + *this = CloneRange(*this); + } + } + + /** Returns a buffer that exactly contains the field range. */ + ZENCORE_API SharedBuffer GetRangeBuffer() const; + +private: + using TCbFieldIterator::TCbFieldIterator; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An array that can hold a reference to the memory that contains it. + * + * @see CbBuffer + */ +class CbArray : public CbBuffer<CbArrayView>, public CbBufferFactory<CbArray, CbArrayView> +{ +public: + using CbBuffer::CbBuffer; + + /** Create an iterator for the fields of this array. */ + inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); } + + /** Access the array as an array field. */ + inline CbField AsField() const& { return CbField(CbArrayView::AsFieldView(), *this); } + + /** Access the array as an array field. */ + inline CbField AsField() && { return CbField(CbArrayView::AsFieldView(), std::move(*this)); } + +private: + friend inline CbFieldIterator begin(const CbArray& Array) { return Array.CreateIterator(); } + friend inline CbFieldIterator end(const CbArray&) { return CbFieldIterator(); } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An object that can hold a reference to the memory that contains it. + * + * @see CbBuffer + */ +class CbObject : public CbBuffer<CbObjectView>, public CbBufferFactory<CbObject, CbObjectView> +{ +public: + using CbBuffer::CbBuffer; + + /** Create an iterator for the fields of this object. */ + inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); } + + /** Find a field by case-sensitive name comparison. */ + inline CbField Find(std::string_view Name) const + { + if (CbFieldView Field = FindView(Name)) + { + return CbField(Field, *this); + } + return CbField(); + } + + /** Find a field by case-insensitive name comparison. */ + inline CbField FindIgnoreCase(std::string_view Name) const + { + if (CbFieldView Field = FindIgnoreCase(Name)) + { + return CbField(Field, *this); + } + return CbField(); + } + + /** Find a field by case-sensitive name comparison. */ + inline CbFieldView operator[](std::string_view Name) const { return Find(Name); } + + /** Access the object as an object field. */ + inline CbField AsField() const& { return CbField(CbObjectView::AsFieldView(), *this); } + + /** Access the object as an object field. */ + inline CbField AsField() && { return CbField(CbObjectView::AsFieldView(), std::move(*this)); } + +private: + friend inline CbFieldIterator begin(const CbObject& Object) { return Object.CreateIterator(); } + friend inline CbFieldIterator end(const CbObject&) { return CbFieldIterator(); } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +inline CbObject +CbField::AsObject() & +{ + return IsObject() ? CbObject(AsObjectView(), *this) : CbObject(); +} + +inline CbObject +CbField::AsObject() && +{ + return IsObject() ? CbObject(AsObjectView(), std::move(*this)) : CbObject(); +} + +inline CbArray +CbField::AsArray() & +{ + return IsArray() ? CbArray(AsArrayView(), *this) : CbArray(); +} + +inline CbArray +CbField::AsArray() && +{ + return IsArray() ? CbArray(AsArrayView(), std::move(*this)) : CbArray(); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator); + +inline CbObject +LoadCompactBinaryObject(IoBuffer Payload) +{ + return CbObject{SharedBuffer::MakeView(Payload.Data(), Payload.Size())}; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Determine the size in bytes of the compact binary field at the start of the view. + * + * This may be called on an incomplete or invalid field, in which case the returned size is zero. + * A size can always be extracted from a valid field with no name if a view of at least the first + * 10 bytes is provided, regardless of field size. For fields with names, the size of view needed + * to calculate a size is at most 10 + MaxNameLen + MeasureVarUInt(MaxNameLen). + * + * This function can be used when streaming a field, for example, to determine the size of buffer + * to fill before attempting to construct a field from it. + * + * @param View A memory view that may contain the start of a field. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + */ +ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType); + +/** + * Try to determine the type and size of the compact binary field at the start of the view. + * + * This may be called on an incomplete or invalid field, in which case it will return false, with + * OutSize being 0 for invalid fields, otherwise the minimum view size necessary to make progress + * in measuring the field on the next call to this function. + * + * @note A return of true from this function does not indicate that the entire field is valid. + * + * @param InView A memory view that may contain the start of a field. + * @param OutType The type (with flags) of the field. None is written until a value is available. + * @param OutSize The total field size for a return of true, 0 for invalid fields, or the size to + * make progress in measuring the field on the next call to this function. + * @param InType HasFieldType means that InView contains the type. Otherwise, use the given type. + * @return true if the size of the field was determined, otherwise false. + */ +ZENCORE_API bool TryMeasureCompactBinary(MemoryView InView, + CbFieldType& OutType, + uint64_t& OutSize, + CbFieldType InType = CbFieldType::HasFieldType); + +inline CbFieldViewIterator +begin(CbFieldView& View) +{ + if (View.IsArray()) + { + return View.AsArrayView().CreateViewIterator(); + } + else if (View.IsObject()) + { + return View.AsObjectView().CreateViewIterator(); + } + + return CbFieldViewIterator(); +} + +inline CbFieldViewIterator +end(CbFieldView&) +{ + return CbFieldViewIterator(); +} + +void uson_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h new file mode 100644 index 000000000..83d4309f7 --- /dev/null +++ b/zencore/include/zencore/compactbinarybuilder.h @@ -0,0 +1,633 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> + +#include <zencore/enumflags.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/refcount.h> +#include <zencore/sha1.h> + +#include <atomic> +#include <memory> +#include <string> +#include <string_view> +#include <type_traits> +#include <vector> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +class CbAttachment; +class BinaryWriter; + +/** + * A writer for compact binary object, arrays, and fields. + * + * The writer produces a sequence of fields that can be saved to a provided memory buffer or into + * a new owned buffer. The typical use case is to write a single object, which can be accessed by + * calling Save().AsObjectRef() or Save(Buffer).AsObject(). + * + * The writer will assert on most incorrect usage and will always produce valid compact binary if + * provided with valid input. The writer does not check for invalid UTF-8 string encoding, object + * fields with duplicate names, or invalid compact binary being copied from another source. + * + * It is most convenient to use the streaming API for the writer, as demonstrated in the example. + * + * When writing a small amount of compact binary data, TCbWriter can be more efficient as it uses + * a fixed-size stack buffer for storage before spilling onto the heap. + * + * @see TCbWriter + * + * Example: + * + * CbObjectRef WriteObject() + * { + * CbWriter<256> Writer; + * Writer.BeginObject(); + * + * Writer << "Resize" << true; + * Writer << "MaxWidth" << 1024; + * Writer << "MaxHeight" << 1024; + * + * Writer.BeginArray(); + * Writer << "FormatA" << "FormatB" << "FormatC"; + * Writer.EndArray(); + * + * Writer.EndObject(); + * return Writer.Save().AsObjectRef(); + * } + */ +class CbWriter +{ +public: + ZENCORE_API CbWriter(); + ZENCORE_API ~CbWriter(); + + CbWriter(const CbWriter&) = delete; + CbWriter& operator=(const CbWriter&) = delete; + + /** Empty the writer without releasing any allocated memory. */ + ZENCORE_API void Reset(); + + /** + * Serialize the field(s) to an owned buffer and return it as an iterator. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + * The writer remains valid for further use when this function returns. + */ + ZENCORE_API CbFieldIterator Save(); + + /** + * Serialize the field(s) to memory. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + * The writer remains valid for further use when this function returns. + * + * @param Buffer A mutable memory view to write to. Must be exactly GetSaveSize() bytes. + * @return An iterator for the field(s) written to the buffer. + */ + ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer); + + ZENCORE_API void Save(BinaryWriter& Writer); + + /** + * The size of buffer (in bytes) required to serialize the fields that have been written. + * + * It is not valid to call this function in the middle of writing an object, array, or field. + */ + ZENCORE_API uint64_t GetSaveSize() const; + + /** + * Sets the name of the next field to be written. + * + * It is not valid to call this function when writing a field inside an array. + * Names must be valid UTF-8 and must be unique within an object. + */ + ZENCORE_API CbWriter& SetName(std::string_view Name); + + /** Copy the value (not the name) of an existing field. */ + inline void AddField(std::string_view Name, const CbFieldView& Value) + { + SetName(Name); + AddField(Value); + } + + ZENCORE_API void AddField(const CbFieldView& Value); + + /** Copy the value (not the name) of an existing field. Holds a reference if owned. */ + inline void AddField(std::string_view Name, const CbField& Value) + { + SetName(Name); + AddField(Value); + } + ZENCORE_API void AddField(const CbField& Value); + + /** Begin a new object. Must have a matching call to EndObject. */ + inline void BeginObject(std::string_view Name) + { + SetName(Name); + BeginObject(); + } + ZENCORE_API void BeginObject(); + /** End an object after its fields have been written. */ + ZENCORE_API void EndObject(); + + /** Copy the value (not the name) of an existing object. */ + inline void AddObject(std::string_view Name, const CbObjectView& Value) + { + SetName(Name); + AddObject(Value); + } + ZENCORE_API void AddObject(const CbObjectView& Value); + /** Copy the value (not the name) of an existing object. Holds a reference if owned. */ + inline void AddObject(std::string_view Name, const CbObject& Value) + { + SetName(Name); + AddObject(Value); + } + ZENCORE_API void AddObject(const CbObject& Value); + + /** Begin a new array. Must have a matching call to EndArray. */ + inline void BeginArray(std::string_view Name) + { + SetName(Name); + BeginArray(); + } + ZENCORE_API void BeginArray(); + /** End an array after its fields have been written. */ + ZENCORE_API void EndArray(); + + /** Copy the value (not the name) of an existing array. */ + inline void AddArray(std::string_view Name, const CbArrayView& Value) + { + SetName(Name); + AddArray(Value); + } + ZENCORE_API void AddArray(const CbArrayView& Value); + /** Copy the value (not the name) of an existing array. Holds a reference if owned. */ + inline void AddArray(std::string_view Name, const CbArray& Value) + { + SetName(Name); + AddArray(Value); + } + ZENCORE_API void AddArray(const CbArray& Value); + + /** Write a null field. */ + inline void AddNull(std::string_view Name) + { + SetName(Name); + AddNull(); + } + ZENCORE_API void AddNull(); + + /** Write a binary field by copying Size bytes from Value. */ + inline void AddBinary(std::string_view Name, const void* Value, uint64_t Size) + { + SetName(Name); + AddBinary(Value, Size); + } + ZENCORE_API void AddBinary(const void* Value, uint64_t Size); + /** Write a binary field by copying the view. */ + inline void AddBinary(std::string_view Name, MemoryView Value) + { + SetName(Name); + AddBinary(Value); + } + inline void AddBinary(MemoryView Value) { AddBinary(Value.GetData(), Value.GetSize()); } + + /** Write a binary field by copying the buffer. Holds a reference if owned. */ + inline void AddBinary(std::string_view Name, IoBuffer Value) + { + SetName(Name); + AddBinary(std::move(Value)); + } + ZENCORE_API void AddBinary(IoBuffer Value); + ZENCORE_API void AddBinary(SharedBuffer Value); + + /** Write a string field by copying the UTF-8 value. */ + inline void AddString(std::string_view Name, std::string_view Value) + { + SetName(Name); + AddString(Value); + } + ZENCORE_API void AddString(std::string_view Value); + /** Write a string field by converting the UTF-16 value to UTF-8. */ + inline void AddString(std::string_view Name, std::wstring_view Value) + { + SetName(Name); + AddString(Value); + } + ZENCORE_API void AddString(std::wstring_view Value); + + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, int32_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(int32_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, int64_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(int64_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, uint32_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(uint32_t Value); + /** Write an integer field. */ + inline void AddInteger(std::string_view Name, uint64_t Value) + { + SetName(Name); + AddInteger(Value); + } + ZENCORE_API void AddInteger(uint64_t Value); + + /** Write a float field from a 32-bit float value. */ + inline void AddFloat(std::string_view Name, float Value) + { + SetName(Name); + AddFloat(Value); + } + ZENCORE_API void AddFloat(float Value); + + /** Write a float field from a 64-bit float value. */ + inline void AddFloat(std::string_view Name, double Value) + { + SetName(Name); + AddFloat(Value); + } + ZENCORE_API void AddFloat(double Value); + + /** Write a bool field. */ + inline void AddBool(std::string_view Name, bool bValue) + { + SetName(Name); + AddBool(bValue); + } + ZENCORE_API void AddBool(bool bValue); + + /** Write a field referencing a compact binary attachment by its hash. */ + inline void AddCompactBinaryAttachment(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddCompactBinaryAttachment(Value); + } + ZENCORE_API void AddCompactBinaryAttachment(const IoHash& Value); + + /** Write a field referencing a binary attachment by its hash. */ + inline void AddBinaryAttachment(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddBinaryAttachment(Value); + } + ZENCORE_API void AddBinaryAttachment(const IoHash& Value); + + /** Write a field referencing the attachment by its hash. */ + inline void AddAttachment(std::string_view Name, const CbAttachment& Attachment) + { + SetName(Name); + AddAttachment(Attachment); + } + ZENCORE_API void AddAttachment(const CbAttachment& Attachment); + + /** Write a hash field. */ + inline void AddHash(std::string_view Name, const IoHash& Value) + { + SetName(Name); + AddHash(Value); + } + ZENCORE_API void AddHash(const IoHash& Value); + + /** Write a UUID field. */ + inline void AddUuid(std::string_view Name, const Guid& Value) + { + SetName(Name); + AddUuid(Value); + } + ZENCORE_API void AddUuid(const Guid& Value); + + /** Write an ObjectId field. */ + inline void AddObjectId(std::string_view Name, const Oid& Value) + { + SetName(Name); + AddObjectId(Value); + } + ZENCORE_API void AddObjectId(const Oid& Value); + + /** Write a date/time field with the specified count of 100ns ticks since the epoch. */ + inline void AddDateTimeTicks(std::string_view Name, int64_t Ticks) + { + SetName(Name); + AddDateTimeTicks(Ticks); + } + ZENCORE_API void AddDateTimeTicks(int64_t Ticks); + + /** Write a date/time field. */ + inline void AddDateTime(std::string_view Name, DateTime Value) + { + SetName(Name); + AddDateTime(Value); + } + ZENCORE_API void AddDateTime(DateTime Value); + + /** Write a time span field with the specified count of 100ns ticks. */ + inline void AddTimeSpanTicks(std::string_view Name, int64_t Ticks) + { + SetName(Name); + AddTimeSpanTicks(Ticks); + } + ZENCORE_API void AddTimeSpanTicks(int64_t Ticks); + + /** Write a time span field. */ + inline void AddTimeSpan(std::string_view Name, TimeSpan Value) + { + SetName(Name); + AddTimeSpan(Value); + } + ZENCORE_API void AddTimeSpan(TimeSpan Value); + + /** Private flags that are public to work with ENUM_CLASS_FLAGS. */ + enum class StateFlags : uint8_t; + +protected: + /** Reserve the specified size up front until the format is optimized. */ + ZENCORE_API explicit CbWriter(int64_t InitialSize); + +private: + friend CbWriter& operator<<(CbWriter& Writer, std::string_view NameOrValue); + + /** Begin writing a field. May be called twice for named fields. */ + void BeginField(); + + /** Finish writing a field by writing its type. */ + void EndField(CbFieldType Type); + + /** Set the field name if valid in this state, otherwise write add a string field. */ + ZENCORE_API void SetNameOrAddString(std::string_view NameOrValue); + + /** Returns a view of the name of the active field, if any, otherwise the empty view. */ + std::string_view GetActiveName() const; + + /** Remove field types after the first to make the sequence uniform. */ + void MakeFieldsUniform(int64_t FieldBeginOffset, int64_t FieldEndOffset); + + /** State of the object, array, or top-level field being written. */ + struct WriterState + { + StateFlags Flags{}; + /** The type of the fields in the sequence if uniform, otherwise None. */ + CbFieldType UniformType{}; + /** The offset of the start of the current field. */ + int64_t Offset{}; + /** The number of fields written in this state. */ + uint64_t Count{}; + }; + +private: + // This is a prototype-quality format for the writer. Using an array of bytes is inefficient, + // and will lead to many unnecessary copies and moves of the data to resize the array, insert + // object and array sizes, and remove field types for uniform objects and uniform arrays. The + // optimized format will be a list of power-of-two blocks and an optional first block that is + // provided externally, such as on the stack. That format will store the offsets that require + // object or array sizes to be inserted and field types to be removed, and will perform those + // operations only when saving to a buffer. + std::vector<uint8_t> Data; + std::vector<WriterState> States; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A writer for compact binary object, arrays, and fields that uses a fixed-size stack buffer. + * + * @see CbWriter + */ +template<uint32_t InlineBufferSize> +class FixedCbWriter : public CbWriter +{ +public: + inline FixedCbWriter() : CbWriter(InlineBufferSize) {} + + FixedCbWriter(const FixedCbWriter&) = delete; + FixedCbWriter& operator=(const FixedCbWriter&) = delete; + +private: + // Reserve the inline buffer now even though we are unable to use it. This will avoid causing + // new stack overflows when this functionality is properly implemented in the future. + uint8_t Buffer[InlineBufferSize]; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class CbObjectWriter : public CbWriter +{ +public: + CbObjectWriter() { BeginObject(); } + + ZENCORE_API CbObject Save() + { + Finalize(); + return CbWriter::Save().AsObject(); + } + + ZENCORE_API void Save(BinaryWriter& Writer) + { + Finalize(); + return CbWriter::Save(Writer); + } + + uint64_t GetSaveSize() = delete; + + void Finalize() + { + if (m_Finalized == false) + { + EndObject(); + m_Finalized = true; + } + } + + CbObjectWriter(const CbWriter&) = delete; + CbObjectWriter& operator=(const CbWriter&) = delete; + +private: + bool m_Finalized = false; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Write the field name if valid in this state, otherwise write the string value. */ +inline CbWriter& +operator<<(CbWriter& Writer, std::string_view NameOrValue) +{ + Writer.SetNameOrAddString(NameOrValue); + return Writer; +} + +/** Write the field name if valid in this state, otherwise write the string value. */ +inline CbWriter& +operator<<(CbWriter& Writer, const char* NameOrValue) +{ + return Writer << std::string_view(NameOrValue); +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbFieldView& Value) +{ + Writer.AddField(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbField& Value) +{ + Writer.AddField(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbObjectView& Value) +{ + Writer.AddObject(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbObject& Value) +{ + Writer.AddObject(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbArrayView& Value) +{ + Writer.AddArray(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbArray& Value) +{ + Writer.AddArray(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, nullptr_t) +{ + Writer.AddNull(); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, std::wstring_view Value) +{ + Writer.AddString(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const wchar_t* Value) +{ + Writer.AddString(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, int32_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, int64_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, uint32_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, uint64_t Value) +{ + Writer.AddInteger(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, float Value) +{ + Writer.AddFloat(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, double Value) +{ + Writer.AddFloat(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, bool Value) +{ + Writer.AddBool(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const CbAttachment& Attachment) +{ + Writer.AddAttachment(Attachment); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const IoHash& Value) +{ + Writer.AddHash(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const Guid& Value) +{ + Writer.AddUuid(Value); + return Writer; +} + +inline CbWriter& +operator<<(CbWriter& Writer, const Oid& Value) +{ + Writer.AddObjectId(Value); + return Writer; +} + +ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value); +ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void usonbuilder_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compactbinarypackage.h b/zencore/include/zencore/compactbinarypackage.h new file mode 100644 index 000000000..c98ab047f --- /dev/null +++ b/zencore/include/zencore/compactbinarypackage.h @@ -0,0 +1,305 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> +#include <zencore/iohash.h> + +#include <functional> +#include <span> + +namespace zen { + +class CbWriter; +class BinaryReader; +class BinaryWriter; +class IoBuffer; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * An attachment is either binary or compact binary and is identified by its hash. + * + * A compact binary attachment is also a valid binary attachment and may be accessed as binary. + * + * Attachments are serialized as one or two compact binary fields with no name. A Binary field is + * written first with its content. The content hash is omitted when the content size is zero, and + * is otherwise written as a BinaryReference or CompactBinaryReference depending on the type. + */ +class CbAttachment +{ +public: + /** Construct a null attachment. */ + CbAttachment() = default; + + /** Construct a compact binary attachment. Value is cloned if not owned. */ + inline explicit CbAttachment(CbFieldIterator Value) : CbAttachment(std::move(Value), nullptr) {} + + /** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */ + inline explicit CbAttachment(CbFieldIterator Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {} + + /** Construct a binary attachment. Value is cloned if not owned. */ + inline explicit CbAttachment(SharedBuffer Value) : CbAttachment(std::move(Value), nullptr) {} + + /** Construct a binary attachment. Value is cloned if not owned. Hash must match Value. */ + inline explicit CbAttachment(SharedBuffer Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {} + + /** Reset this to a null attachment. */ + inline void Reset() { *this = CbAttachment(); } + + /** Whether the attachment has a value. */ + inline explicit operator bool() const { return !IsNull(); } + + /** Whether the attachment has a value. */ + inline bool IsNull() const { return !Buffer; } + + /** Access the attachment as binary. Defaults to a null buffer on error. */ + ZENCORE_API SharedBuffer AsBinaryView() const; + + /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ + ZENCORE_API CbFieldIterator AsCompactBinary() const; + + /** Returns whether the attachment is binary or compact binary. */ + inline bool IsBinary() const { return !Buffer.IsNull(); } + + /** Returns whether the attachment is compact binary. */ + inline bool IsCompactBinary() const { return CompactBinary.HasValue(); } + + /** Returns the hash of the attachment value. */ + inline const IoHash& GetHash() const { return Hash; } + + /** Compares attachments by their hash. Any discrepancy in type must be handled externally. */ + inline bool operator==(const CbAttachment& Attachment) const { return Hash == Attachment.Hash; } + inline bool operator!=(const CbAttachment& Attachment) const { return Hash != Attachment.Hash; } + inline bool operator<(const CbAttachment& Attachment) const { return Hash < Attachment.Hash; } + + /** + * Load the attachment from compact binary as written by Save. + * + * The attachment references the input iterator if it is owned, and otherwise clones the value. + * + * The iterator is advanced as attachment fields are consumed from it. + */ + ZENCORE_API void Load(CbFieldIterator& Fields); + + /** + * Load the attachment from compact binary as written by Save. + */ + ZENCORE_API void Load(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc); + + /** + * Load the attachment from compact binary as written by Save. + */ + ZENCORE_API void Load(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc); + + /** Save the attachment into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(CbWriter& Writer) const; + + /** Save the attachment into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(BinaryWriter& Writer) const; + +private: + ZENCORE_API CbAttachment(CbFieldIterator Value, const IoHash* Hash); + ZENCORE_API CbAttachment(SharedBuffer Value, const IoHash* Hash); + + /** An owned buffer containing the binary or compact binary data. */ + SharedBuffer Buffer; + /** A field iterator that is valid only for compact binary attachments. */ + CbFieldViewIterator CompactBinary; + /** A hash of the attachment value. */ + IoHash Hash; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A package is a compact binary object with attachments for its external references. + * + * A package is basically a Merkle tree with compact binary as its root and other non-leaf nodes, + * and either binary or compact binary as its leaf nodes. A node references its child nodes using + * BinaryHash or FieldHash fields in its compact binary representation. + * + * It is invalid for a package to include attachments that are not referenced by its object or by + * one of its referenced compact binary attachments. When attachments are added explicitly, it is + * the responsibility of the package creator to follow this requirement. Attachments that are not + * referenced may not survive a round-trip through certain storage systems. + * + * It is valid for a package to exclude referenced attachments, but then it is the responsibility + * of the package consumer to have a mechanism for resolving those references when necessary. + * + * A package is serialized as a sequence of compact binary fields with no name. The object may be + * both preceded and followed by attachments. The object itself is written as an Object field and + * followed by its hash in a CompactBinaryReference field when the object is non-empty. A package + * ends with a Null field. The canonical order of components is the object and its hash, followed + * by the attachments ordered by hash, followed by a Null field. It is valid for the a package to + * have its components serialized in any order, provided there is at most one object and the null + * field is written last. + */ +class CbPackage +{ +public: + /** + * A function that resolves a hash to a buffer containing the data matching that hash. + * + * The resolver may return a null buffer to skip resolving an attachment for the hash. + */ + using AttachmentResolver = std::function<SharedBuffer(const IoHash& Hash)>; + + /** Construct a null package. */ + CbPackage() = default; + + /** + * Construct a package from a root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + */ + inline explicit CbPackage(CbObject InObject) { SetObject(std::move(InObject)); } + + /** + * Construct a package from a root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline explicit CbPackage(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), InResolver); } + + /** + * Construct a package from a root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + */ + inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), InObjectHash); } + + /** + * Construct a package from a root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver) + { + SetObject(std::move(InObject), InObjectHash, InResolver); + } + + /** Reset this to a null package. */ + inline void Reset() { *this = CbPackage(); } + + /** Whether the package has a non-empty object or attachments. */ + inline explicit operator bool() const { return !IsNull(); } + + /** Whether the package has an empty object and no attachments. */ + inline bool IsNull() const { return !Object.CreateIterator() && Attachments.size() == 0; } + + /** Returns the compact binary object for the package. */ + inline const CbObject& GetObject() const { return Object; } + + /** Returns the has of the compact binary object for the package. */ + inline const IoHash& GetObjectHash() const { return ObjectHash; } + + /** + * Set the root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + */ + inline void SetObject(CbObject InObject) { SetObject(std::move(InObject), nullptr, nullptr); } + + /** + * Set the root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline void SetObject(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), nullptr, &InResolver); } + + /** + * Set the root object without gathering attachments. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + */ + inline void SetObject(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), &InObjectHash, nullptr); } + + /** + * Set the root object and gather attachments using the resolver. + * + * @param InObject The root object, which will be cloned unless it is owned. + * @param InObjectHash The hash of the object, which must match to avoid validation errors. + * @param InResolver A function that is invoked for every reference and binary reference field. + */ + inline void SetObject(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver) + { + SetObject(std::move(InObject), &InObjectHash, &InResolver); + } + + /** Returns the attachments in this package. */ + inline std::span<const CbAttachment> GetAttachments() const { return Attachments; } + + /** + * Find an attachment by its hash. + * + * @return The attachment, or null if the attachment is not found. + * @note The returned pointer is only valid until the attachments on this package are modified. + */ + ZENCORE_API const CbAttachment* FindAttachment(const IoHash& Hash) const; + + /** Find an attachment if it exists in the package. */ + inline const CbAttachment* FindAttachment(const CbAttachment& Attachment) const { return FindAttachment(Attachment.GetHash()); } + + /** Add the attachment to this package. */ + inline void AddAttachment(const CbAttachment& Attachment) { AddAttachment(Attachment, nullptr); } + + /** Add the attachment to this package, along with any references that can be resolved. */ + inline void AddAttachment(const CbAttachment& Attachment, AttachmentResolver Resolver) { AddAttachment(Attachment, &Resolver); } + + /** + * Remove an attachment by hash. + * + * @return Number of attachments removed, which will be either 0 or 1. + */ + ZENCORE_API int32_t RemoveAttachment(const IoHash& Hash); + inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); } + + /** Compares packages by their object and attachment hashes. */ + ZENCORE_API bool Equals(const CbPackage& Package) const; + inline bool operator==(const CbPackage& Package) const { return Equals(Package); } + inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); } + + /** + * Load the object and attachments from compact binary as written by Save. + * + * The object and attachments reference the input iterator, if it is owned, and otherwise clones + * the object and attachments individually to make owned copies. + * + * The iterator is advanced as object and attachment fields are consumed from it. + */ + ZENCORE_API void Load(CbFieldIterator& Fields); + + ZENCORE_API void Load(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc); + + ZENCORE_API void Load(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc); + + /** Save the object and attachments into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(CbWriter& Writer) const; + + /** Save the object and attachments into the writer as a stream of compact binary fields. */ + ZENCORE_API void Save(BinaryWriter& Writer) const; + +private: + ZENCORE_API void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver); + ZENCORE_API void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver); + + void GatherAttachments(const CbFieldViewIterator& Fields, AttachmentResolver Resolver); + + /** Attachments ordered by their hash. */ + std::vector<CbAttachment> Attachments; + CbObject Object; + IoHash ObjectHash; +}; + +void usonpackage_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compactbinaryvalidation.h b/zencore/include/zencore/compactbinaryvalidation.h new file mode 100644 index 000000000..3a3f432be --- /dev/null +++ b/zencore/include/zencore/compactbinaryvalidation.h @@ -0,0 +1,192 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#include <zencore/compactbinary.h> +#include <zencore/enumflags.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/refcount.h> +#include <zencore/sha1.h> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +/** Flags for validating compact binary data. */ +enum class CbValidateMode : uint32_t +{ + /** Skip validation if no other validation modes are enabled. */ + None = 0, + + /** + * Validate that the value can be read and stays inside the bounds of the memory view. + * + * This is the minimum level of validation required to be able to safely read a field, array, + * or object without the risk of crashing or reading out of bounds. + */ + Default = 1 << 0, + + /** + * Validate that object fields have unique non-empty names and array fields have no names. + * + * Name validation failures typically do not inhibit reading the input, but duplicated fields + * cannot be looked up by name other than the first, and converting to other data formats can + * fail in the presence of naming issues. + */ + Names = 1 << 1, + + /** + * Validate that fields are serialized in the canonical format. + * + * Format validation failures typically do not inhibit reading the input. Values that fail in + * this mode require more memory than in the canonical format, and comparisons of such values + * for equality are not reliable. Examples of failures include uniform arrays or objects that + * were not encoded uniformly, variable-length integers that could be encoded in fewer bytes, + * or 64-bit floats that could be encoded in 32 bits without loss of precision. + */ + Format = 1 << 2, + + /** + * Validate that there is no padding after the value before the end of the memory view. + * + * Padding validation failures have no impact on the ability to read the input, but are using + * more memory than necessary. + */ + Padding = 1 << 3, + + /** + * Validate that a package or attachment has the expected fields and matches its saved hashes. + */ + Package = 1 << 4, + + /** Perform all validation described above. */ + All = Default | Names | Format | Padding | Package, +}; + +ENUM_CLASS_FLAGS(CbValidateMode); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Flags for compact binary validation errors. Multiple flags may be combined. */ +enum class CbValidateError : uint32_t +{ + /** The input had no validation errors. */ + None = 0, + + // Mode: Default + + /** The input cannot be read without reading out of bounds. */ + OutOfBounds = 1 << 0, + /** The input has a field with an unrecognized or invalid type. */ + InvalidType = 1 << 1, + + // Mode: Names + + /** An object had more than one field with the same name. */ + DuplicateName = 1 << 2, + /** An object had a field with no name. */ + MissingName = 1 << 3, + /** An array field had a name. */ + ArrayName = 1 << 4, + + // Mode: Format + + /** A name or string payload is not valid UTF-8. */ + InvalidString = 1 << 5, + /** A size or integer payload can be encoded in fewer bytes. */ + InvalidInteger = 1 << 6, + /** A float64 payload can be encoded as a float32 without loss of precision. */ + InvalidFloat = 1 << 7, + /** An object has the same type for every field but is not uniform. */ + NonUniformObject = 1 << 8, + /** An array has the same type for every field and non-empty payloads but is not uniform. */ + NonUniformArray = 1 << 9, + + // Mode: Padding + + /** A value did not use the entire memory view given for validation. */ + Padding = 1 << 10, + + // Mode: Package + + /** The package or attachment had missing fields or fields out of order. */ + InvalidPackageFormat = 1 << 11, + /** The object or an attachment did not match the hash stored for it. */ + InvalidPackageHash = 1 << 12, + /** The package contained more than one copy of the same attachment. */ + DuplicateAttachments = 1 << 13, + /** The package contained more than one object. */ + MultiplePackageObjects = 1 << 14, + /** The package contained an object with no fields. */ + NullPackageObject = 1 << 15, + /** The package contained a null attachment. */ + NullPackageAttachment = 1 << 16, +}; + +ENUM_CLASS_FLAGS(CbValidateError); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Validate the compact binary data for one field in the view as specified by the mode flags. + * + * Only one top-level field is processed from the view, and validation recurses into any array or + * object within that field. To validate multiple consecutive top-level fields, call the function + * once for each top-level field. If the given view might contain multiple top-level fields, then + * either exclude the Padding flag from the Mode or use MeasureCompactBinary to break up the view + * into its constituent fields before validating. + * + * @param View A memory view containing at least one top-level field. + * @param Mode A combination of the flags for the types of validation to perform. + * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType); + +/** + * Validate the compact binary data for every field in the view as specified by the mode flags. + * + * This function expects the entire view to contain fields. Any trailing region of the view which + * does not contain a valid field will produce an OutOfBounds or InvalidType error instead of the + * Padding error that would be produced by the single field validation function. + * + * @see ValidateCompactBinary + */ +ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode); + +/** + * Validate the compact binary attachment pointed to by the view as specified by the mode flags. + * + * The attachment is validated with ValidateCompactBinary by using the validation mode specified. + * Include ECbValidateMode::Package to validate the attachment format and hash. + * + * @see ValidateCompactBinary + * + * @param View A memory view containing a package. + * @param Mode A combination of the flags for the types of validation to perform. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateCompactBinaryAttachment(MemoryView View, CbValidateMode Mode); + +/** + * Validate the compact binary package pointed to by the view as specified by the mode flags. + * + * The package, and attachments, are validated with ValidateCompactBinary by using the validation + * mode specified. Include ECbValidateMode::Package to validate the package format and hashes. + * + * @see ValidateCompactBinary + * + * @param View A memory view containing a package. + * @param Mode A combination of the flags for the types of validation to perform. + * @return None on success, otherwise the flags for the types of errors that were detected. + */ +ZENCORE_API CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void usonvalidation_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h new file mode 100644 index 000000000..759cf8444 --- /dev/null +++ b/zencore/include/zencore/compress.h @@ -0,0 +1,53 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore/zencore.h" + +namespace zen::CompressedBuffer { + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static constexpr uint64_t DefaultBlockSize = 256 * 1024; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Method used to compress the data in a compressed buffer. */ +enum class Method : uint8_t +{ + /** Header is followed by one uncompressed block. */ + None = 0, + /** Header is followed by an array of compressed block sizes then the compressed blocks. */ + LZ4 = 4, +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Header used on every compressed buffer. Always stored in big-endian format */ +struct BufferHeader +{ + static constexpr uint32_t ExpectedMagic = 0xb7756362; + + /** A magic number to identify a compressed buffer. Always 0xb7756362 */ + uint32_t Magic = ExpectedMagic; + /** A CRC-32 used to check integrity of the buffer. Uses the polynomial 0x04c11db7 */ + uint32_t Crc32 = 0; + /** The method used to compress the buffer. Affects layout of data following the header */ + Method Method = Method::None; + /** The reserved bytes must be initialized to zero */ + uint8_t Reserved[2]{}; + /** The power of two size of every uncompressed block except the last. Size is 1 << BlockSizeExponent */ + uint8_t BlockSizeExponent = 0; + /** The number of blocks that follow the header */ + uint32_t BlockCount = 0; + /** The total size of the uncompressed data */ + uint64_t TotalRawSize = 0; + /** The total size of the compressed data including the header */ + uint64_t TotalCompressedSize = 0; +}; + +static_assert(sizeof(BufferHeader) == 32, "BufferHeader is the wrong size"); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace zen::CompressedBuffer diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h new file mode 100644 index 000000000..27c831bb1 --- /dev/null +++ b/zencore/include/zencore/endian.h @@ -0,0 +1,61 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +namespace zen { + +inline uint16_t +ByteSwap(uint16_t x) +{ + return _byteswap_ushort(x); +} + +inline uint32_t +ByteSwap(uint32_t x) +{ + return _byteswap_ulong(x); +} + +inline uint64_t +ByteSwap(uint64_t x) +{ + return _byteswap_uint64(x); +} + +inline uint16_t +FromNetworkOrder(uint16_t x) +{ + return ByteSwap(x); +} + +inline uint32_t +FromNetworkOrder(uint32_t x) +{ + return ByteSwap(x); +} + +inline uint64_t +FromNetworkOrder(uint64_t x) +{ + return ByteSwap(x); +} + +inline uint16_t +FromNetworkOrder(int16_t x) +{ + return ByteSwap(uint16_t(x)); +} + +inline uint32_t +FromNetworkOrder(int32_t x) +{ + return ByteSwap(uint32_t(x)); +} + +inline uint64_t +FromNetworkOrder(int64_t x) +{ + return ByteSwap(uint64_t(x)); +} + +} // namespace zen diff --git a/zencore/include/zencore/enumflags.h b/zencore/include/zencore/enumflags.h new file mode 100644 index 000000000..ebe747bf0 --- /dev/null +++ b/zencore/include/zencore/enumflags.h @@ -0,0 +1,61 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +namespace zen { + +// Enum class helpers + +// Defines all bitwise operators for enum classes so it can be (mostly) used as a regular flags enum +#define ENUM_CLASS_FLAGS(Enum) \ + inline Enum& operator|=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \ + inline Enum& operator&=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \ + inline Enum& operator^=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator|(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator&(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \ + inline constexpr Enum operator^(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \ + inline constexpr bool operator!(Enum E) { return !(__underlying_type(Enum))E; } \ + inline constexpr Enum operator~(Enum E) { return (Enum) ~(__underlying_type(Enum))E; } + +// Friends all bitwise operators for enum classes so the definition can be kept private / protected. +#define FRIEND_ENUM_CLASS_FLAGS(Enum) \ + friend Enum& operator|=(Enum& Lhs, Enum Rhs); \ + friend Enum& operator&=(Enum& Lhs, Enum Rhs); \ + friend Enum& operator^=(Enum& Lhs, Enum Rhs); \ + friend constexpr Enum operator|(Enum Lhs, Enum Rhs); \ + friend constexpr Enum operator&(Enum Lhs, Enum Rhs); \ + friend constexpr Enum operator^(Enum Lhs, Enum Rhs); \ + friend constexpr bool operator!(Enum E); \ + friend constexpr Enum operator~(Enum E); + +template<typename Enum> +constexpr bool +EnumHasAllFlags(Enum Flags, Enum Contains) +{ + return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) == ((__underlying_type(Enum))Contains); +} + +template<typename Enum> +constexpr bool +EnumHasAnyFlags(Enum Flags, Enum Contains) +{ + return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) != 0; +} + +template<typename Enum> +void +EnumAddFlags(Enum& Flags, Enum FlagsToAdd) +{ + Flags |= FlagsToAdd; +} + +template<typename Enum> +void +EnumRemoveFlags(Enum& Flags, Enum FlagsToRemove) +{ + Flags &= ~FlagsToRemove; +} + +} // namespace zen diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h new file mode 100644 index 000000000..07c4833ff --- /dev/null +++ b/zencore/include/zencore/except.h @@ -0,0 +1,60 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/string.h> +#include <zencore/windows.h> +#include <string> + +namespace zen { + +class WindowsException : public std::exception +{ +public: + WindowsException(const char* Message) + { + m_hResult = HRESULT_FROM_WIN32(GetLastError()); + m_Message = Message; + } + + WindowsException(HRESULT hRes, const char* Message) + { + m_hResult = hRes; + m_Message = Message; + } + + WindowsException(HRESULT hRes, const char* Message, const char* Detail) + { + m_hResult = hRes; + + ExtendableStringBuilder<128> msg; + msg.Append(Message); + msg.Append(" (detail: '"); + msg.Append(Detail); + msg.Append("')"); + + m_Message = msg.c_str(); + } + + virtual const char* what() const override { return m_Message.c_str(); } + +private: + std::string m_Message; + HRESULT m_hResult; +}; + +ZENCORE_API void ThrowSystemException(HRESULT hRes, const char* Message); +inline void +ThrowSystemException(const char* Message) +{ + throw WindowsException(Message); +} + +inline void +ThrowIfFailed(HRESULT hRes, const char* Message) +{ + if (FAILED(hRes)) + ThrowSystemException(hRes, Message); +} + +} // namespace zen diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h new file mode 100644 index 000000000..b20a1e7c6 --- /dev/null +++ b/zencore/include/zencore/filesystem.h @@ -0,0 +1,74 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "stream.h" +#include "zencore.h" + +#include <filesystem> +#include <functional> + +namespace zen { + +class IoBuffer; + +/** Delete directory (after deleting any contents) + */ +ZENCORE_API bool DeleteDirectories(const wchar_t* dir); +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); + +/** Ensure directory exists. + + Will also create any required parent directories + */ +ZENCORE_API bool CreateDirectories(const wchar_t* dir); +ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); + +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectory(const wchar_t* dir); +ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); + +struct FileContents +{ + std::vector<IoBuffer> Data; + std::error_code ErrorCode; +}; + +ZENCORE_API FileContents ReadFile(std::filesystem::path Path); +ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc); +ZENCORE_API bool WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount); + +struct CopyFileOptions +{ + bool EnableClone = true; + bool MustClone = false; +}; + +ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); +ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); + +ZENCORE_API std::string ToUtf8(const std::filesystem::path& Path); + +/** + * Efficient file system traversal + * + * Uses the best available mechanism for the platform in question and could take + * advantage of any file system tracking mechanisms in the future + * + */ +class FileSystemTraversal +{ +public: + struct TreeVisitor + { + virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) = 0; + + // This should return true if we should recurse into the directory + virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) = 0; + }; + + void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor); +}; + +} // namespace zen diff --git a/zencore/include/zencore/fmtutils.h b/zencore/include/zencore/fmtutils.h new file mode 100644 index 000000000..fb5a08d56 --- /dev/null +++ b/zencore/include/zencore/fmtutils.h @@ -0,0 +1,49 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/iohash.h> +#include <zencore/string.h> +#include <zencore/uid.h> + +#include <fmt/format.h> +#include <filesystem> +#include <string_view> + +// Custom formatting for some zencore types + +template<> +struct fmt::formatter<zen::IoHash> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const zen::IoHash& Hash, FormatContext& ctx) + { + zen::IoHash::String_t String; + Hash.ToHexString(String); + return formatter<string_view>::format({String, zen::IoHash::StringLength}, ctx); + } +}; + +template<> +struct fmt::formatter<zen::Oid> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const zen::Oid& Id, FormatContext& ctx) + { + zen::StringBuilder<32> String; + Id.ToString(String); + return formatter<string_view>::format({String.c_str(), zen::Oid::StringLength}, ctx); + } +}; + +template<> +struct fmt::formatter<std::filesystem::path> : formatter<string_view> +{ + template<typename FormatContext> + auto format(const std::filesystem::path& Path, FormatContext& ctx) + { + zen::ExtendableStringBuilder<128> String; + WideToUtf8(Path.c_str(), String); + return formatter<string_view>::format(String.ToView(), ctx); + } +}; diff --git a/zencore/include/zencore/httpclient.h b/zencore/include/zencore/httpclient.h new file mode 100644 index 000000000..4ede6839c --- /dev/null +++ b/zencore/include/zencore/httpclient.h @@ -0,0 +1,18 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/string.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +class HttpClient +{ +}; + +} // namespace zen + +void httpclient_forcelink(); // internal diff --git a/zencore/include/zencore/httpserver.h b/zencore/include/zencore/httpserver.h new file mode 100644 index 000000000..563245264 --- /dev/null +++ b/zencore/include/zencore/httpserver.h @@ -0,0 +1,373 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/enumflags.h> +#include <zencore/refcount.h> +#include <zencore/string.h> +#include <functional> +#include <gsl/gsl-lite.hpp> +#include <list> +#include <regex> +#include <span> +#include <unordered_map> +#include "zencore.h" + +namespace zen { + +class IoBuffer; +class CbObject; +class StringBuilderBase; + +enum class HttpVerb +{ + kGet = 1 << 0, + kPut = 1 << 1, + kPost = 1 << 2, + kDelete = 1 << 3, + kHead = 1 << 4, + kCopy = 1 << 5, + kOptions = 1 << 6 +}; + +gsl_DEFINE_ENUM_BITMASK_OPERATORS(HttpVerb); + +enum class HttpResponse +{ + // 1xx - Informational + + Continue = 100, //!< Indicates that the initial part of a request has been received and has not yet been rejected by the server. + SwitchingProtocols = 101, //!< Indicates that the server understands and is willing to comply with the client's request, via the + //!< Upgrade header field, for a change in the application protocol being used on this connection. + Processing = 102, //!< Is an interim response used to inform the client that the server has accepted the complete request, but has not + //!< yet completed it. + EarlyHints = 103, //!< Indicates to the client that the server is likely to send a final response with the header fields included in + //!< the informational response. + + // 2xx - Successful + + OK = 200, //!< Indicates that the request has succeeded. + Created = 201, //!< Indicates that the request has been fulfilled and has resulted in one or more new resources being created. + Accepted = 202, //!< Indicates that the request has been accepted for processing, but the processing has not been completed. + NonAuthoritativeInformation = 203, //!< Indicates that the request was successful but the enclosed payload has been modified from that + //!< of the origin server's 200 (OK) response by a transforming proxy. + NoContent = 204, //!< Indicates that the server has successfully fulfilled the request and that there is no additional content to send + //!< in the response payload body. + ResetContent = 205, //!< Indicates that the server has fulfilled the request and desires that the user agent reset the \"document + //!< view\", which caused the request to be sent, to its original state as received from the origin server. + PartialContent = 206, //!< Indicates that the server is successfully fulfilling a range request for the target resource by transferring + //!< one or more parts of the selected representation that correspond to the satisfiable ranges found in the + //!< requests's Range header field. + MultiStatus = 207, //!< Provides status for multiple independent operations. + AlreadyReported = 208, //!< Used inside a DAV:propstat response element to avoid enumerating the internal members of multiple bindings + //!< to the same collection repeatedly. [RFC 5842] + IMUsed = 226, //!< The server has fulfilled a GET request for the resource, and the response is a representation of the result of one + //!< or more instance-manipulations applied to the current instance. + + // 3xx - Redirection + + MultipleChoices = 300, //!< Indicates that the target resource has more than one representation, each with its own more specific + //!< identifier, and information about the alternatives is being provided so that the user (or user agent) can + //!< select a preferred representation by redirecting its request to one or more of those identifiers. + MovedPermanently = 301, //!< Indicates that the target resource has been assigned a new permanent URI and any future references to this + //!< resource ought to use one of the enclosed URIs. + Found = 302, //!< Indicates that the target resource resides temporarily under a different URI. + SeeOther = 303, //!< Indicates that the server is redirecting the user agent to a different resource, as indicated by a URI in the + //!< Location header field, that is intended to provide an indirect response to the original request. + NotModified = 304, //!< Indicates that a conditional GET request has been received and would have resulted in a 200 (OK) response if it + //!< were not for the fact that the condition has evaluated to false. + UseProxy = 305, //!< \deprecated \parblock Due to security concerns regarding in-band configuration of a proxy. \endparblock + //!< The requested resource MUST be accessed through the proxy given by the Location field. + TemporaryRedirect = 307, //!< Indicates that the target resource resides temporarily under a different URI and the user agent MUST NOT + //!< change the request method if it performs an automatic redirection to that URI. + PermanentRedirect = 308, //!< The target resource has been assigned a new permanent URI and any future references to this resource + //!< ought to use one of the enclosed URIs. [...] This status code is similar to 301 Moved Permanently + //!< (Section 7.3.2 of rfc7231), except that it does not allow rewriting the request method from POST to GET. + + // 4xx - Client Error + BadRequest = 400, //!< Indicates that the server cannot or will not process the request because the received syntax is invalid, + //!< nonsensical, or exceeds some limitation on what the server is willing to process. + Unauthorized = 401, //!< Indicates that the request has not been applied because it lacks valid authentication credentials for the + //!< target resource. + PaymentRequired = 402, //!< *Reserved* + Forbidden = 403, //!< Indicates that the server understood the request but refuses to authorize it. + NotFound = 404, //!< Indicates that the origin server did not find a current representation for the target resource or is not willing + //!< to disclose that one exists. + MethodNotAllowed = 405, //!< Indicates that the method specified in the request-line is known by the origin server but not supported by + //!< the target resource. + NotAcceptable = 406, //!< Indicates that the target resource does not have a current representation that would be acceptable to the + //!< user agent, according to the proactive negotiation header fields received in the request, and the server is + //!< unwilling to supply a default representation. + ProxyAuthenticationRequired = + 407, //!< Is similar to 401 (Unauthorized), but indicates that the client needs to authenticate itself in order to use a proxy. + RequestTimeout = + 408, //!< Indicates that the server did not receive a complete request message within the time that it was prepared to wait. + Conflict = 409, //!< Indicates that the request could not be completed due to a conflict with the current state of the resource. + Gone = 410, //!< Indicates that access to the target resource is no longer available at the origin server and that this condition is + //!< likely to be permanent. + LengthRequired = 411, //!< Indicates that the server refuses to accept the request without a defined Content-Length. + PreconditionFailed = + 412, //!< Indicates that one or more preconditions given in the request header fields evaluated to false when tested on the server. + PayloadTooLarge = 413, //!< Indicates that the server is refusing to process a request because the request payload is larger than the + //!< server is willing or able to process. + URITooLong = 414, //!< Indicates that the server is refusing to service the request because the request-target is longer than the + //!< server is willing to interpret. + UnsupportedMediaType = 415, //!< Indicates that the origin server is refusing to service the request because the payload is in a format + //!< not supported by the target resource for this method. + RangeNotSatisfiable = 416, //!< Indicates that none of the ranges in the request's Range header field overlap the current extent of the + //!< selected resource or that the set of ranges requested has been rejected due to invalid ranges or an + //!< excessive request of small or overlapping ranges. + ExpectationFailed = 417, //!< Indicates that the expectation given in the request's Expect header field could not be met by at least + //!< one of the inbound servers. + ImATeapot = 418, //!< Any attempt to brew coffee with a teapot should result in the error code 418 I'm a teapot. + UnprocessableEntity = 422, //!< Means the server understands the content type of the request entity (hence a 415(Unsupported Media + //!< Type) status code is inappropriate), and the syntax of the request entity is correct (thus a 400 (Bad + //!< Request) status code is inappropriate) but was unable to process the contained instructions. + Locked = 423, //!< Means the source or destination resource of a method is locked. + FailedDependency = 424, //!< Means that the method could not be performed on the resource because the requested action depended on + //!< another action and that action failed. + UpgradeRequired = 426, //!< Indicates that the server refuses to perform the request using the current protocol but might be willing to + //!< do so after the client upgrades to a different protocol. + PreconditionRequired = 428, //!< Indicates that the origin server requires the request to be conditional. + TooManyRequests = 429, //!< Indicates that the user has sent too many requests in a given amount of time (\"rate limiting\"). + RequestHeaderFieldsTooLarge = + 431, //!< Indicates that the server is unwilling to process the request because its header fields are too large. + UnavailableForLegalReasons = + 451, //!< This status code indicates that the server is denying access to the resource in response to a legal demand. + + // 5xx - Server Error + + InternalServerError = + 500, //!< Indicates that the server encountered an unexpected condition that prevented it from fulfilling the request. + NotImplemented = 501, //!< Indicates that the server does not support the functionality required to fulfill the request. + BadGateway = 502, //!< Indicates that the server, while acting as a gateway or proxy, received an invalid response from an inbound + //!< server it accessed while attempting to fulfill the request. + ServiceUnavailable = 503, //!< Indicates that the server is currently unable to handle the request due to a temporary overload or + //!< scheduled maintenance, which will likely be alleviated after some delay. + GatewayTimeout = 504, //!< Indicates that the server, while acting as a gateway or proxy, did not receive a timely response from an + //!< upstream server it needed to access in order to complete the request. + HTTPVersionNotSupported = 505, //!< Indicates that the server does not support, or refuses to support, the protocol version that was + //!< used in the request message. + VariantAlsoNegotiates = + 506, //!< Indicates that the server has an internal configuration error: the chosen variant resource is configured to engage in + //!< transparent content negotiation itself, and is therefore not a proper end point in the negotiation process. + InsufficientStorage = 507, //!< Means the method could not be performed on the resource because the server is unable to store the + //!< representation needed to successfully complete the request. + LoopDetected = 508, //!< Indicates that the server terminated an operation because it encountered an infinite loop while processing a + //!< request with "Depth: infinity". [RFC 5842] + NotExtended = 510, //!< The policy for accessing the resource has not been met in the request. [RFC 2774] + NetworkAuthenticationRequired = 511, //!< Indicates that the client needs to authenticate to gain network access. +}; + +enum class HttpContentType +{ + kBinary, + kText, + kJSON, + kCbObject, + kCbPackage +}; + +/** HTTP Server Request + */ +class HttpServerRequest +{ +public: + HttpServerRequest(); + ~HttpServerRequest(); + + // Synchronous operations + + inline [[nodiscard]] std::string_view RelativeUri() const { return m_Uri; } // Returns URI without service prefix + inline [[nodiscard]] std::string_view QueryString() const { return m_QueryString; } + inline bool IsHandled() const { return m_IsHandled; } + + struct QueryParams + { + std::vector<std::pair<std::string_view, std::string_view>> KvPairs; + + std::string_view GetValue(std::string_view ParamName) + { + for (const auto& Kv : KvPairs) + { + const std::string_view& Key = Kv.first; + + if (Key.size() == ParamName.size()) + { + if (0 == _strnicmp(Key.data(), ParamName.data(), Key.size())) + { + return Kv.second; + } + } + } + + return std::string_view(); + } + }; + + QueryParams GetQueryParams(); + + inline HttpVerb RequestVerb() const { return m_Verb; } + + const char* HeaderAccept() const; + const char* HeaderAcceptEncoding() const; + const char* HeaderContentType() const; + const char* HeaderContentEncoding() const; + inline uint64_t HeaderContentLength() const { return m_ContentLength; } + + void SetSuppressResponseBody() { m_SuppressBody = true; } + + // Asynchronous operations + + /** Read POST/PUT payload + + This will return a null buffer if the contents are not fully available yet, and the handler should + at that point return - another completion request will be issues once the contents have been received + fully. + */ + virtual IoBuffer ReadPayload() = 0; + + /** Respond with payload + + Note that this is destructive in the sense that the IoBuffer instances referred to by Blobs will be + moved into our response handler array where they are kept alive, in order to reduce ref-counting storms + */ + virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) = 0; + virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, IoBuffer Blob); + virtual void WriteResponse(HttpResponse HttpResponseCode) = 0; + + virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) = 0; + + void WriteResponse(HttpResponse HttpResponseCode, CbObject Data); + void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::string_view ResponseString); + +protected: + bool m_IsHandled = false; + bool m_SuppressBody = false; + HttpVerb m_Verb = HttpVerb::kGet; + uint64_t m_ContentLength = ~0ull; + ExtendableStringBuilder<256> m_Uri; + ExtendableStringBuilder<256> m_QueryString; +}; + +class HttpServerException : public std::exception +{ +public: + HttpServerException(const char* Message, uint32_t Error) : std::exception(Message), m_ErrorCode(Error) {} + +private: + uint32_t m_ErrorCode; +}; + +class HttpService +{ +public: + HttpService() = default; + virtual ~HttpService() = default; + + virtual const char* BaseUri() const = 0; + virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) = 0; + + // Internals + + inline void SetUriPrefixLength(size_t PrefixLength) { m_UriPrefixLength = (int)PrefixLength; } + inline int UriPrefixLength() const { return m_UriPrefixLength; } + +private: + int m_UriPrefixLength = 0; +}; + +/** HTTP server + */ +class HttpServer +{ +public: + HttpServer(); + ~HttpServer(); + + void AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler); + void AddEndpoint(HttpService& Service); + + void Initialize(int BasePort); + void Run(bool TestMode); + void RequestExit(); + +private: + struct Impl; + + RefPtr<Impl> m_Impl; +}; + +////////////////////////////////////////////////////////////////////////// + +class HttpRouterRequest +{ +public: + HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {} + + ZENCORE_API std::string GetCapture(int Index) const; + inline HttpServerRequest& ServerRequest() { return m_HttpRequest; } + +private: + using MatchResults_t = std::match_results<std::string_view::const_iterator>; + + HttpServerRequest& m_HttpRequest; + MatchResults_t m_Match; + + friend class HttpRequestRouter; +}; + +inline std::string +HttpRouterRequest::GetCapture(int Index) const +{ + ZEN_ASSERT(Index < m_Match.size()); + + return m_Match[Index]; +} + +////////////////////////////////////////////////////////////////////////// + +class HttpRequestRouter +{ +public: + typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t; + + void AddPattern(const char* Id, const char* Regex); + void RegisterRoute(const char* Regex, HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs); + bool HandleRequest(zen::HttpServerRequest& Request); + +private: + struct HandlerEntry + { + HandlerEntry(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern) + : RegEx(Regex, std::regex::icase | std::regex::ECMAScript) + , Verbs(SupportedVerbs) + , Handler(std::move(Handler)) + , Pattern(Pattern) + { + } + + ~HandlerEntry() = default; + + std::regex RegEx; + HttpVerb Verbs; + HandlerFunc_t Handler; + const char* Pattern; + }; + + std::list<HandlerEntry> m_Handlers; + std::unordered_map<std::string, std::string> m_PatternMap; +}; + +////////////////////////////////////////////////////////////////////////// +// +// HTTP Client +// + +class HttpClient +{ +}; + +} // namespace zen + +void http_forcelink(); // internal diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h new file mode 100644 index 000000000..9d39b8226 --- /dev/null +++ b/zencore/include/zencore/intmath.h @@ -0,0 +1,140 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <stdint.h> +#include <concepts> + +////////////////////////////////////////////////////////////////////////// + +#pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanReverse64) + +namespace zen { + +inline constexpr bool +IsPow2(uint64_t n) +{ + return 0 == (n & (n - 1)); +} + +/// Round an integer up to the closest integer multiplier of 'base' ('base' must be a power of two) +template<std::integral T> +T +RoundUp(T Value, auto Base) +{ + ZEN_ASSERT_SLOW(IsPow2(Base)); + return ((Value + T(Base - 1)) & (~T(Base - 1))); +} + +bool +IsMultipleOf(std::integral auto Value, auto MultiplierPow2) +{ + ZEN_ASSERT_SLOW(IsPow2(MultiplierPow2)); + return (Value & (MultiplierPow2 - 1)) == 0; +} + +inline uint64_t +NextPow2(uint64_t n) +{ + // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + + --n; + + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n |= n >> 32; + + return n + 1; +} + +static inline uint32_t +FloorLog2(uint32_t Value) +{ + // Use BSR to return the log2 of the integer + unsigned long Log2; + if (_BitScanReverse(&Log2, Value) != 0) + { + return Log2; + } + + return 0; +} + +static inline uint32_t +CountLeadingZeros(uint32_t Value) +{ + unsigned long Log2; + _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1); + return 32 - Log2; +} + +static inline uint64_t +FloorLog2_64(uint64_t Value) +{ + unsigned long Log2; + long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); + return Log2 & Mask; +} + +static inline uint64_t +CountLeadingZeros64(uint64_t Value) +{ + unsigned long Log2; + long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); + return ((63 - Log2) & Mask) | (64 & ~Mask); +} + +static inline uint64_t +CeilLogTwo64(uint64_t Arg) +{ + int64_t Bitmask = ((int64_t)(CountLeadingZeros64(Arg) << 57)) >> 63; + return (64 - CountLeadingZeros64(Arg - 1)) & (~Bitmask); +} + +static inline uint64_t +CountTrailingZeros64(uint64_t Value) +{ + if (Value == 0) + { + return 64; + } + unsigned long BitIndex; // 0-based, where the LSB is 0 and MSB is 31 + _BitScanForward64(&BitIndex, Value); // Scans from LSB to MSB + return BitIndex; +} + +////////////////////////////////////////////////////////////////////////// + +static inline bool +IsPointerAligned(const void* Ptr, uint64_t Alignment) +{ + ZEN_ASSERT_SLOW(IsPow2(Alignment)); + + return 0 == (reinterpret_cast<uintptr_t>(Ptr) & (Alignment - 1)); +} + +////////////////////////////////////////////////////////////////////////// + +#ifdef min +# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead" +#endif + +auto +Min(auto x, auto y) +{ + return x < y ? x : y; +} + +auto +Max(auto x, auto y) +{ + return x > y ? x : y; +} + +} // namespace zen diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h new file mode 100644 index 000000000..c93d27959 --- /dev/null +++ b/zencore/include/zencore/iobuffer.h @@ -0,0 +1,272 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <memory.h> +#include "refcount.h" +#include "zencore.h" + +namespace zen { + +struct IoBufferExtendedCore; + +struct IoBufferFileReference +{ + void* FileHandle; + uint64_t FileChunkOffset; + uint64_t FileChunkSize; +}; + +struct IoBufferCore +{ +public: + IoBufferCore() = default; + inline IoBufferCore(const void* DataPtr, size_t SizeBytes) : m_DataPtr(DataPtr), m_DataBytes(SizeBytes) {} + inline IoBufferCore(const IoBufferCore* Outer, const void* DataPtr, size_t SizeBytes) + : m_DataPtr(DataPtr) + , m_DataBytes(SizeBytes) + , m_OuterCore(Outer) + { + } + + ZENCORE_API explicit IoBufferCore(size_t SizeBytes); + ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment); + ZENCORE_API ~IoBufferCore(); + + // Reference counting + + inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); } + inline uint32_t Release() const + { + const uint32_t NewRefCount = AtomicDecrement(const_cast<IoBufferCore*>(this)->m_RefCount); + if (NewRefCount == 0) + { + DeleteThis(); + } + return NewRefCount; + } + + // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it + + IoBufferCore(const IoBufferCore&) = delete; + IoBufferCore(IoBufferCore&&) = delete; + IoBufferCore& operator=(const IoBufferCore&) = delete; + IoBufferCore& operator=(IoBufferCore&&) = delete; + + // + + ZENCORE_API void Materialize() const; + ZENCORE_API void DeleteThis() const; + ZENCORE_API void MakeOwned(bool Immutable = true); + + inline void EnsureDataValid() const + { + if ((m_Flags & kIsExtended) && !(m_Flags & kIsMaterialized)) + Materialize(); + } + + inline bool IsOwned() const { return !!(m_Flags & kIsOwned); } + inline bool IsImmutable() const { return !(m_Flags & kIsMutable); } + inline bool IsNull() const { return m_DataBytes == 0; } + + inline IoBufferExtendedCore* ExtendedCore(); + inline const IoBufferExtendedCore* ExtendedCore() const; + + inline const void* DataPointer() const + { + EnsureDataValid(); + return m_DataPtr; + } + + inline size_t DataBytes() const { return m_DataBytes; } + + inline void Set(const void* Ptr, size_t Sz) + { + m_DataPtr = Ptr; + m_DataBytes = Sz; + } + + inline void SetIsOwned(bool NewState) + { + if (NewState) + { + m_Flags |= kIsOwned; + } + else + { + m_Flags &= ~kIsOwned; + } + } + + inline void SetIsImmutable(bool NewState) + { + if (!NewState) + { + m_Flags |= kIsMutable; + } + else + { + m_Flags &= ~kIsMutable; + } + } + + inline uint32_t GetRefCount() const { return m_RefCount; } + +protected: + uint32_t m_RefCount = 0; + mutable uint32_t m_Flags = 0; + mutable const void* m_DataPtr = nullptr; + size_t m_DataBytes = 0; + RefPtr<const IoBufferCore> m_OuterCore; + + enum Flags + { + kIsOwned = 1 << 0, + kIsMutable = 1 << 1, + kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore + kIsMaterialized = 1 << 3, // Data pointers are valid + kLowLevelAlloc = 1 << 4, // Using direct memory allocation + }; + + void* AllocateBuffer(size_t InSize, size_t Alignment); + void FreeBuffer(); +}; + +/** + * An "Extended" core references a segment of a file + */ + +struct IoBufferExtendedCore : public IoBufferCore +{ + IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership); + IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size); + ~IoBufferExtendedCore(); + + enum ExtendedFlags + { + kOwnsFile = 1 << 8, + kOwnsMmap = 1 << 9 + }; + + void Materialize() const; + bool GetFileReference(IoBufferFileReference& OutRef) const; + +private: + void* m_FileHandle = nullptr; + uint64_t m_FileOffset = 0; + mutable void* m_MmapHandle = nullptr; + mutable void* m_MappedPointer = nullptr; +}; + +inline IoBufferExtendedCore* +IoBufferCore::ExtendedCore() +{ + if (m_Flags & kIsExtended) + { + return static_cast<IoBufferExtendedCore*>(this); + } + + return nullptr; +} + +inline const IoBufferExtendedCore* +IoBufferCore::ExtendedCore() const +{ + if (m_Flags & kIsExtended) + { + return static_cast<const IoBufferExtendedCore*>(this); + } + + return nullptr; +} + +/** + * I/O buffer + * + * This represents a reference to a payload in memory or on disk + * + */ +class IoBuffer +{ +public: + enum EAssumeOwnershipTag + { + AssumeOwnership + }; + enum ECloneTag + { + Clone + }; + enum EWrapTag + { + Wrap + }; + enum EFileTag + { + File + }; + enum EBorrowedFileTag + { + BorrowedFile + }; + + inline IoBuffer() = default; + inline IoBuffer(IoBuffer&& Rhs) noexcept = default; + inline IoBuffer(const IoBuffer& Rhs) = default; + inline IoBuffer& operator=(const IoBuffer& Rhs) = default; + inline IoBuffer& operator=(IoBuffer&& Rhs) noexcept = default; + + /** Create an uninitialized buffer of the given size + */ + ZENCORE_API explicit IoBuffer(size_t InSize); + + /** Create an uninitialized buffer of the given size with the specified alignment + */ + ZENCORE_API explicit IoBuffer(size_t InSize, uint64_t InAlignment); + + /** Create a buffer which references a sequence of bytes inside another buffer + */ + ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes); + + /** Create a buffer which references a range of bytes which we assume will live + * for the entire life time. + */ + inline IoBuffer(EWrapTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(DataPtr, SizeBytes)) {} + + inline IoBuffer(ECloneTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(SizeBytes)) + { + memcpy(const_cast<void*>(m_Core->DataPointer()), DataPtr, SizeBytes); + } + + inline IoBuffer(EAssumeOwnershipTag, const void* DataPtr, size_t Sz) : m_Core(new IoBufferCore(DataPtr, Sz)) + { + m_Core->SetIsOwned(true); + } + + ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); + ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); + + inline operator bool() const { return !m_Core->IsNull(); } + ZENCORE_API void MakeOwned() { return m_Core->MakeOwned(); } + inline bool IsOwned() const { return m_Core->IsOwned(); } + const void* Data() const { return m_Core->DataPointer(); } + const size_t Size() const { return m_Core->DataBytes(); } + ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const; + +private: + RefPtr<IoBufferCore> m_Core = new IoBufferCore; +}; + +class IoBufferBuilder +{ +public: + ZENCORE_API static IoBuffer MakeFromFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); + inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); } + +private: +}; + +void iobuffer_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/iohash.h b/zencore/include/zencore/iohash.h new file mode 100644 index 000000000..4eac7e328 --- /dev/null +++ b/zencore/include/zencore/iohash.h @@ -0,0 +1,95 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/blake3.h> +#include <zencore/memory.h> + +#include <string_view> + +namespace zen { + +class StringBuilderBase; + +/** + * Hash used for content addressable storage + * + * This is basically a BLAKE3-160 hash (note: this is probably not an officially + * recognized identifier). It is generated by computing a 32-byte BLAKE3 hash and + * picking the first 20 bytes of the resulting hash. + * + */ +struct IoHash +{ + uint8_t Hash[20]; + + static IoHash MakeFrom(const void* data /* 20 bytes */) + { + IoHash Io; + memcpy(Io.Hash, data, sizeof Io); + return Io; + } + + static IoHash HashMemory(const void* data, size_t byteCount); + static IoHash HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); } + static IoHash FromHexString(const char* string); + static IoHash FromHexString(const std::string_view string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + std::string ToHexString() const; + + static const int StringLength = 40; + typedef char String_t[StringLength + 1]; + + static IoHash Zero; // Initialized to all zeros + + inline auto operator<=>(const IoHash& rhs) const = default; + + struct Hasher + { + size_t operator()(const IoHash& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct IoHashStream +{ + /// Begin streaming hash compute (not needed on freshly constructed instance) + void Reset() { m_Blake3Stream.Reset(); } + + /// Append another chunk + IoHashStream& Append(const void* data, size_t byteCount) + { + m_Blake3Stream.Append(data, byteCount); + return *this; + } + + /// Append another chunk + IoHashStream& Append(MemoryView Data) + { + m_Blake3Stream.Append(Data.GetData(), Data.GetSize()); + return *this; + } + + /// Obtain final hash. If you wish to reuse the instance call reset() + IoHash GetHash() + { + BLAKE3 b3 = m_Blake3Stream.GetHash(); + + IoHash Io; + memcpy(Io.Hash, b3.Hash, sizeof Io.Hash); + + return Io; + } + +private: + BLAKE3Stream m_Blake3Stream; +}; + +} // namespace zen diff --git a/zencore/include/zencore/md5.h b/zencore/include/zencore/md5.h new file mode 100644 index 000000000..4ed4f6c56 --- /dev/null +++ b/zencore/include/zencore/md5.h @@ -0,0 +1,50 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <stdint.h> +#include <compare> +#include "zencore.h" + +namespace zen { + +class StringBuilderBase; + +struct MD5 +{ + uint8_t Hash[16]; + + inline auto operator<=>(const MD5& rhs) const = default; + + static const int StringLength = 40; + typedef char String_t[StringLength + 1]; + + static MD5 HashMemory(const void* data, size_t byteCount); + static MD5 FromHexString(const char* string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static MD5 Zero; // Initialized to all zeroes +}; + +/** + * Utility class for computing SHA1 hashes + */ +class MD5Stream +{ +public: + MD5Stream(); + + /// Begin streaming MD5 compute (not needed on freshly constructed MD5Stream instance) + void Reset(); + /// Append another chunk + MD5Stream& Append(const void* data, size_t byteCount); + /// Obtain final MD5 hash. If you wish to reuse the MD5Stream instance call reset() + MD5 GetHash(); + +private: +}; + +void md5_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h new file mode 100644 index 000000000..8a16126ef --- /dev/null +++ b/zencore/include/zencore/memory.h @@ -0,0 +1,213 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/thread.h> + +#include <algorithm> +#include <vector> + +namespace zen { + +class MemoryArena +{ +public: + ZENCORE_API MemoryArena(); + ZENCORE_API ~MemoryArena(); + + ZENCORE_API void* Alloc(size_t size, size_t alignment); + ZENCORE_API void Free(void* ptr); + +private: +}; + +class Memory +{ +public: + ZENCORE_API static void* Alloc(size_t size, size_t alignment = sizeof(void*)); + ZENCORE_API static void Free(void* ptr); +}; + +/** Allocator which claims fixed-size blocks from the underlying allocator. + + There is no way to free individual memory blocks. + + \note This is not thread-safe, you will need to provide synchronization yourself +*/ + +class ChunkingLinearAllocator +{ +public: + ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment = sizeof(std::max_align_t)); + ~ChunkingLinearAllocator(); + + ZENCORE_API void Reset(); + + ZENCORE_API void* Alloc(size_t Size, size_t Alignment = sizeof(void*)); + inline void Free(void* Ptr) { ZEN_UNUSED(Ptr); /* no-op */ } + +private: + uint8_t* m_ChunkCursor = nullptr; + uint64_t m_ChunkBytesRemain = 0; + const uint64_t m_ChunkSize = 0; + const uint64_t m_ChunkAlignment = 0; + std::vector<void*> m_ChunkList; +}; + +////////////////////////////////////////////////////////////////////////// + +struct MutableMemoryView +{ + MutableMemoryView() = default; + + MutableMemoryView(void* DataPtr, size_t DataSize) + : m_Data(reinterpret_cast<uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<uint8_t*>(DataPtr) + DataSize) + { + } + + inline bool IsEmpty() const { return m_Data == m_DataEnd; } + void* GetData() const { return m_Data; } + void* GetDataEnd() const { return m_DataEnd; } + size_t GetSize() const { return reinterpret_cast<uint8_t*>(m_DataEnd) - reinterpret_cast<uint8_t*>(m_Data); } + + inline bool EqualBytes(const MutableMemoryView& InView) const + { + const size_t Size = GetSize(); + + return Size == InView.GetSize() && (memcmp(m_Data, InView.m_Data, Size) == 0); + } + + /** Modifies the view by chopping the given number of bytes from the left. */ + inline void RightChopInline(uint64_t InSize) + { + const uint64_t Offset = std::min(GetSize(), InSize); + m_Data = GetDataAtOffsetNoCheck(Offset); + } + + /** Returns the left-most part of the view by taking the given number of bytes from the left. */ + constexpr inline MutableMemoryView Left(uint64_t InSize) const + { + MutableMemoryView View(*this); + View.LeftInline(InSize); + return View; + } + + /** Modifies the view to be the given number of bytes from the left. */ + constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = std::min(m_DataEnd, m_Data + InSize); } + +private: + uint8_t* m_Data = nullptr; + uint8_t* m_DataEnd = nullptr; + + /** Returns the data pointer advanced by an offset in bytes. */ + inline uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; } +}; + +struct MemoryView +{ + MemoryView() = default; + + MemoryView(const MutableMemoryView& MutableView) + : m_Data(reinterpret_cast<const uint8_t*>(MutableView.GetData())) + , m_DataEnd(m_Data + MutableView.GetSize()) + { + } + + MemoryView(const void* DataPtr, size_t DataSize) + : m_Data(reinterpret_cast<const uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<const uint8_t*>(DataPtr) + DataSize) + { + } + + MemoryView(const void* DataPtr, const void* DataEndPtr) + : m_Data(reinterpret_cast<const uint8_t*>(DataPtr)) + , m_DataEnd(reinterpret_cast<const uint8_t*>(DataEndPtr)) + { + } + + inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); } + inline bool IsEmpty() const { return m_Data == m_DataEnd; } + const void* GetData() const { return m_Data; } + const void* GetDataEnd() const { return m_DataEnd; } + size_t GetSize() const { return reinterpret_cast<const uint8_t*>(m_DataEnd) - reinterpret_cast<const uint8_t*>(m_Data); } + inline bool operator==(const MemoryView& Rhs) const { return m_Data == Rhs.m_Data && m_DataEnd == Rhs.m_DataEnd; } + + inline bool EqualBytes(const MemoryView& InView) const + { + const size_t Size = GetSize(); + + return Size == InView.GetSize() && (memcmp(m_Data, InView.GetData(), Size) == 0); + } + + inline MemoryView& operator+=(size_t InSize) + { + RightChopInline(InSize); + return *this; + } + + /** Modifies the view by chopping the given number of bytes from the left. */ + inline void RightChopInline(uint64_t InSize) + { + const uint64_t Offset = std::min(GetSize(), InSize); + m_Data = GetDataAtOffsetNoCheck(Offset); + } + + inline MemoryView RightChop(uint64_t InSize) + { + MemoryView View(*this); + View.RightChopInline(InSize); + return View; + } + + /** Returns the left-most part of the view by taking the given number of bytes from the left. */ + constexpr inline MemoryView Left(uint64_t InSize) const + { + MemoryView View(*this); + View.LeftInline(InSize); + return View; + } + + /** Modifies the view to be the given number of bytes from the left. */ + constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = std::min(m_DataEnd, m_Data + InSize); } + + constexpr void Reset() + { + m_Data = nullptr; + m_DataEnd = nullptr; + } + +private: + const uint8_t* m_Data = nullptr; + const uint8_t* m_DataEnd = nullptr; + + /** Returns the data pointer advanced by an offset in bytes. */ + inline const uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; } +}; + +/** + * Make a non-owning view of the memory of the initializer list. + * + * This overload is only available when the element type does not need to be deduced. + */ +template<typename T> +[[nodiscard]] inline MemoryView +MakeMemoryView(std::initializer_list<typename std::type_identity<T>::type> List) +{ + return MemoryView(List.begin(), List.size() * sizeof(T)); +} + +/** Make a non-owning view of the memory of the contiguous container. */ +template<std::ranges::contiguous_range R> +[[nodiscard]] constexpr inline MemoryView +MakeMemoryView(const R& Container) +{ + const auto& Front = *std::begin(Container); + return MemoryView(std::addressof(Front), std::size(Container) * sizeof(Front)); +} + +void memory_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/meta.h b/zencore/include/zencore/meta.h new file mode 100644 index 000000000..82eb5cc30 --- /dev/null +++ b/zencore/include/zencore/meta.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +/* This file contains utility functions for meta programming + * + * Since you're in here you're probably quite observant, and you'll + * note that it's quite barren here. This is because template + * metaprogramming is awful and I try not to engage in it. However, + * sometimes these things are forced upon us. + * + */ + +namespace zen { + +/** + * Uses implicit conversion to create an instance of a specific type. + * Useful to make things clearer or circumvent unintended type deduction in templates. + * Safer than C casts and static_casts, e.g. does not allow down-casts + * + * @param Obj The object (usually pointer or reference) to convert. + * + * @return The object converted to the specified type. + */ +template<typename T> +inline T +ImplicitConv(typename std::type_identity<T>::type Obj) +{ + return Obj; +} + +} // namespace zen diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h new file mode 100644 index 000000000..de7d315f4 --- /dev/null +++ b/zencore/include/zencore/refcount.h @@ -0,0 +1,144 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +#pragma once + +#include "atomic.h" +#include "zencore.h" + +namespace zen { + +/** + * Helper base class for reference counted objects using intrusive reference counts + * + * This class is pretty straightforward but does one thing which may be unexpected: + * + * - Instances on the stack are initialized with a reference count of one to ensure + * nobody tries to accidentally delete it. (TODO: is this really useful?) + */ +class RefCounted +{ +public: + RefCounted() : m_RefCount(IsPointerToStack(this)){}; + virtual ~RefCounted() = default; + + inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); } + inline uint32_t Release() const + { + uint32_t refCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount); + if (refCount == 0) + { + delete this; + } + return refCount; + } + + // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it + + RefCounted(const RefCounted&) = delete; + RefCounted(RefCounted&&) = delete; + RefCounted& operator=(const RefCounted&) = delete; + RefCounted& operator=(RefCounted&&) = delete; + +protected: + inline uint32_t RefCount() const { return m_RefCount; } + +private: + uint32_t m_RefCount = 0; +}; + +/** + * Smart pointer for classes derived from RefCounted + */ + +template<class T> +class RefPtr +{ +public: + inline RefPtr() = default; + inline RefPtr(const RefPtr& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } + inline RefPtr(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } + inline ~RefPtr() { m_Ref && m_Ref->Release(); } + + inline explicit operator bool() const { return m_Ref != nullptr; } + inline operator T*() const { return m_Ref; } + inline T* operator->() const { return m_Ref; } + + inline std::strong_ordering operator<=>(const RefPtr& Rhs) const = default; + + inline RefPtr& operator=(T* Rhs) + { + Rhs && Rhs->AddRef(); + m_Ref && m_Ref->Release(); + m_Ref = Rhs; + return *this; + } + inline RefPtr& operator=(const RefPtr& Rhs) + { + m_Ref && m_Ref->Release(); + auto Ref = m_Ref = Rhs.m_Ref; + Ref && Ref->AddRef(); + return *this; + } + inline RefPtr& operator=(RefPtr&& Rhs) noexcept + { + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + Rhs.m_Ref = nullptr; + return *this; + } + inline RefPtr(RefPtr&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } + +private: + T* m_Ref = nullptr; +}; + +/** + * Smart pointer for classes derived from RefCounted + * + * This variant does not decay to a raw pointer + * + */ + +template<class T> +class Ref +{ +public: + inline Ref() = default; + inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } + inline Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } + inline ~Ref() { m_Ref && m_Ref->Release(); } + + inline explicit operator bool() const { return m_Ref != nullptr; } + inline T* operator->() const { return m_Ref; } + + inline std::strong_ordering operator<=>(const Ref& Rhs) const = default; + + inline Ref& operator=(T* Rhs) + { + Rhs && Rhs->AddRef(); + m_Ref && m_Ref->Release(); + m_Ref = Rhs; + return *this; + } + inline Ref& operator=(const Ref& Rhs) + { + m_Ref && m_Ref->Release(); + auto Ref = m_Ref = Rhs.m_Ref; + Ref && Ref->AddRef(); + return *this; + } + inline Ref& operator=(Ref&& Rhs) noexcept + { + m_Ref && m_Ref->Release(); + m_Ref = Rhs.m_Ref; + Rhs.m_Ref = nullptr; + return *this; + } + inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } + +private: + T* m_Ref = nullptr; +}; + +void refcount_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/scopeguard.h b/zencore/include/zencore/scopeguard.h new file mode 100644 index 000000000..ba8cd3094 --- /dev/null +++ b/zencore/include/zencore/scopeguard.h @@ -0,0 +1,33 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <type_traits> +#include "zencore.h" + +namespace zen { + +template<typename T> +class ScopeGuardImpl +{ +public: + inline ScopeGuardImpl(T&& func) : m_guardFunc(func) {} + ~ScopeGuardImpl() + { + if (!m_dismissed) + m_guardFunc(); + } + + void Dismiss() { m_dismissed = true; } + +private: + bool m_dismissed = false; + T m_guardFunc; +}; + +template<typename T> +ScopeGuardImpl<T> +MakeGuard(T&& fn) +{ + return ScopeGuardImpl<T>(std::move(fn)); +} + +} // namespace zen diff --git a/zencore/include/zencore/sha1.h b/zencore/include/zencore/sha1.h new file mode 100644 index 000000000..fc26f442b --- /dev/null +++ b/zencore/include/zencore/sha1.h @@ -0,0 +1,76 @@ +// ////////////////////////////////////////////////////////// +// sha1.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +#include <stdint.h> +#include <compare> +#include "zencore.h" + +namespace zen { + +class StringBuilderBase; + +struct SHA1 +{ + uint8_t Hash[20]; + + inline auto operator<=>(const SHA1& rhs) const = default; + + static const int StringLength = 40; + typedef char String_t[StringLength + 1]; + + static SHA1 HashMemory(const void* data, size_t byteCount); + static SHA1 FromHexString(const char* string); + const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static SHA1 Zero; // Initialized to all zeroes +}; + +/** + * Utility class for computing SHA1 hashes + */ +class SHA1Stream +{ +public: + SHA1Stream(); + + /** compute SHA1 of a memory block + + \note SHA1 class contains a slightly more convenient helper function for this use case + \see SHA1::fromMemory() + */ + SHA1 Compute(const void* data, size_t byteCount); + + /// Begin streaming SHA1 compute (not needed on freshly constructed SHA1Stream instance) + void Reset(); + /// Append another chunk + SHA1Stream& Append(const void* data, size_t byteCount); + /// Obtain final SHA1 hash. If you wish to reuse the SHA1Stream instance call reset() + SHA1 GetHash(); + +private: + void ProcessBlock(const void* data); + void ProcessBuffer(); + + enum + { + /// split into 64 byte blocks (=> 512 bits) + BlockSize = 512 / 8, + HashBytes = 20, + HashValues = HashBytes / 4 + }; + + uint64_t m_NumBytes; // size of processed data in bytes + size_t m_BufferSize; // valid bytes in m_buffer + uint8_t m_Buffer[BlockSize]; // bytes not processed yet + uint32_t m_Hash[HashValues]; +}; + +void sha1_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h new file mode 100644 index 000000000..c6206f780 --- /dev/null +++ b/zencore/include/zencore/sharedbuffer.h @@ -0,0 +1,169 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/memory.h> +#include <zencore/refcount.h> + +#include <memory.h> + +namespace zen { + +class BufferOwner : public RefCounted +{ +protected: + inline BufferOwner(void* DataPtr, uint64_t DataSize, bool IsOwned, BufferOwner* OuterBuffer = nullptr) + : m_IsOwned(IsOwned) + , m_Data(DataPtr) + , m_Size(DataSize) + , m_Outer(OuterBuffer) + { + } + + virtual ~BufferOwner(); + + // Ownership is a transitive property, and m_IsOwned currently only flags that this instance is responsible + // for managing the allocated memory, so we need to make recursive calls. Could be optimized slightly by + // adding a dedicated flag + inline bool IsOwned() const + { + if (m_IsOwned) + { + return true; + } + else + { + return m_Outer && m_Outer->IsOwned(); + } + } + + BufferOwner(const BufferOwner&) = delete; + BufferOwner& operator=(const BufferOwner&) = delete; + +private: + bool m_IsOwned; + void* m_Data; + uint64_t m_Size; + RefPtr<BufferOwner> m_Outer; + + friend class UniqueBuffer; + friend class SharedBuffer; +}; + +/** + * Reference to a memory buffer with a single owner (see std::unique_ptr) + */ +class UniqueBuffer +{ +public: + UniqueBuffer(const UniqueBuffer&) = delete; + UniqueBuffer& operator=(const UniqueBuffer&) = delete; + + UniqueBuffer() = default; + ZENCORE_API explicit UniqueBuffer(BufferOwner* Owner); + + void* GetData() { return m_buffer->m_Data; } + const void* GetData() const { return m_buffer->m_Data; } + size_t GetSize() const { return m_buffer->m_Size; } + operator MutableMemoryView() { return GetView(); } + operator MemoryView() const { return MemoryView(m_buffer->m_Data, m_buffer->m_Size); } + + MutableMemoryView GetView() { return MutableMemoryView(m_buffer->m_Data, m_buffer->m_Size); } + + /** Make an uninitialized owned buffer of the specified size. */ + ZENCORE_API static UniqueBuffer Alloc(uint64_t Size); + + /** Make a non-owned view of the input. */ + ZENCORE_API static UniqueBuffer MakeView(void* DataPtr, uint64_t Size); + +private: + RefPtr<BufferOwner> m_buffer; + + friend class SharedBuffer; +}; + +/** + * Reference to a memory buffer with shared ownership + */ +class SharedBuffer +{ +public: + SharedBuffer() = default; + ZENCORE_API explicit SharedBuffer(UniqueBuffer&&); + inline explicit SharedBuffer(BufferOwner* Owner) : m_buffer(Owner) {} + + void* GetData() + { + if (m_buffer) + { + return m_buffer->m_Data; + } + return nullptr; + } + + const void* GetData() const + { + if (m_buffer) + { + return m_buffer->m_Data; + } + return nullptr; + } + + size_t GetSize() const + { + if (m_buffer) + { + return m_buffer->m_Size; + } + return 0; + } + + ZENCORE_API void MakeOwned(); + bool IsOwned() const { return m_buffer && m_buffer->IsOwned(); } + inline explicit operator bool() const { return m_buffer; } + inline bool IsNull() const { return !m_buffer; } + inline void Reset() { m_buffer = nullptr; } + + MemoryView GetView() const + { + if (m_buffer) + { + return MemoryView(m_buffer->m_Data, m_buffer->m_Size); + } + else + { + return MemoryView(); + } + } + + operator MemoryView() const { return GetView(); } + + SharedBuffer& operator=(UniqueBuffer&& Rhs) + { + m_buffer = std::move(Rhs.m_buffer); + return *this; + } + + std::strong_ordering operator<=>(const SharedBuffer& Rhs) const = default; + + /** Make a non-owned view of the input */ + inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); } + /** Make a non-owned view of the input */ + ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size); + /** Make a non-owned view of the input */ + ZENCORE_API static SharedBuffer MakeView(MemoryView View, SharedBuffer Buffer); + /** Make am owned clone of the buffer */ + ZENCORE_API SharedBuffer Clone(); + /** Make an owned clone of the memory in the input view */ + ZENCORE_API static SharedBuffer Clone(MemoryView View); + +private: + RefPtr<BufferOwner> m_buffer; +}; + +void sharedbuffer_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/snapshot_manifest.h b/zencore/include/zencore/snapshot_manifest.h new file mode 100644 index 000000000..95e64773a --- /dev/null +++ b/zencore/include/zencore/snapshot_manifest.h @@ -0,0 +1,57 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/iohash.h> +#include <zencore/zencore.h> + +#include <filesystem> +#include <functional> +#include <string> +#include <vector> + +namespace zen { + +struct LeafNode +{ + uint64_t FileSize = 0; + uint64_t FileModifiedTime = 0; + zen::IoHash ChunkHash = zen::IoHash::Zero; + std::wstring Name; +}; + +struct TreeNode +{ + std::vector<TreeNode> Children; + std::vector<LeafNode> Leaves; + std::wstring Name; + zen::BLAKE3 ChunkHash = zen::BLAKE3::Zero; + + ZENCORE_API void VisitModifyFiles(std::function<void(LeafNode& node)> func); + ZENCORE_API void VisitFiles(std::function<void(const LeafNode& node)> func); + ZENCORE_API void Finalize(); +}; + +struct SnapshotManifest +{ + std::string Id; + TreeNode Root; + zen::BLAKE3 ChunkHash = zen::BLAKE3::Zero; + + ZENCORE_API void finalize(); +}; + +class InStream; +class OutStream; + +ZENCORE_API void ReadManifest(SnapshotManifest& Manifest, InStream& FromStream); +ZENCORE_API void WriteManifest(const SnapshotManifest& Manifest, OutStream& ToStream); +ZENCORE_API void PrintManifest(const SnapshotManifest& Manifest, OutStream& ToStream); + +// Translate a user-provided manifest specification into a file path. +// Supports hashtag syntax to implicitly refer to user documents zenfs folder +ZENCORE_API std::filesystem::path ManifestSpecToPath(const char* ManifestSpec); + +void snapshotmanifest_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h new file mode 100644 index 000000000..7290fd914 --- /dev/null +++ b/zencore/include/zencore/stats.h @@ -0,0 +1,66 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <atomic> +#include <type_traits> +#include "zencore.h" + +namespace zen { + +template<typename T> +class Gauge +{ +public: + Gauge() : m_value{0} {} + +private: + T m_value; +}; + +class Counter +{ +public: + inline void SetValue(uint64_t Value) { m_count = Value; } + inline uint64_t Value() const { return m_count; } + + inline void Increment(int64_t AddValue) { m_count += AddValue; } + inline void Decrement(int64_t SubValue) { m_count -= SubValue; } + inline void Clear() { m_count = 0; } + +private: + std::atomic_uint64_t m_count{0}; +}; + +/// <summary> +/// Exponential Weighted Moving Average +/// </summary> +class EWMA +{ +public: + /// <summary> + /// Update EWMA with new measure + /// </summary> + /// <param name="Alpha">Smoothing factor (between 0 and 1)</param> + /// <param name="Interval">Elapsed time since last</param> + /// <param name="Count">Value</param> + /// <param name="IsInitialUpdate">Whether this is the first update or not</param> + void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate); + double Rate() const; + +private: + double m_rate = 0; +}; + +/// <summary> +/// Tracks rate of events over time (i.e requests/sec) +/// </summary> +class Meter +{ +public: +private: +}; + +extern void stats_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/stream.h b/zencore/include/zencore/stream.h new file mode 100644 index 000000000..4e8c58382 --- /dev/null +++ b/zencore/include/zencore/stream.h @@ -0,0 +1,318 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/memory.h> +#include <zencore/refcount.h> +#include <zencore/thread.h> + +#include <string_view> +#include <vector> + +namespace zen { + +/** + * Basic byte stream interface + * + * This is intended as a minimal base class offering only the absolute minimum of functionality. + * + * IMPORTANT: To better support concurrency, this abstraction offers no "file pointer". Thus + * every read or write operation needs to specify the offset from which they wish to read. + * + * Most client code will likely want to use reader/writer classes like BinaryWriter/BinaryReader + * + */ +class OutStream : public RefCounted +{ +public: + virtual void Write(const void* Data, size_t ByteCount, uint64_t Offset) = 0; + virtual void Flush() = 0; +}; + +class InStream : public RefCounted +{ +public: + virtual void Read(void* DataPtr, size_t ByteCount, uint64_t Offset) = 0; + virtual uint64_t Size() const = 0; +}; + +/** + * Stream which writes into a growing memory buffer + */ +class MemoryOutStream : public OutStream +{ +public: + MemoryOutStream() = default; + ~MemoryOutStream() = default; + + virtual void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset) override; + virtual void Flush() override; + inline const uint8_t* Data() const { return m_Buffer.data(); } + inline uint64_t Size() const { return m_Buffer.size(); } + +private: + RwLock m_Lock; + std::vector<uint8_t> m_Buffer; +}; + +inline MemoryView +MakeMemoryView(const MemoryOutStream& Stream) +{ + return MemoryView(Stream.Data(), Stream.Size()); +} + +/** + * Stream which reads from a memory buffer + */ +class MemoryInStream : public InStream +{ +public: + MemoryInStream(const void* Buffer, size_t Size); + MemoryInStream(MemoryView View) : MemoryInStream(View.GetData(), View.GetSize()) {} + ~MemoryInStream() = default; + + virtual void Read(void* DataPtr, size_t ByteCount, uint64_t ReadOffset) override; + virtual uint64_t Size() const override { return m_Buffer.size(); } + inline const uint8_t* Data() const { return m_Buffer.data(); } + +private: + RwLock m_Lock; + std::vector<uint8_t> m_Buffer; +}; + +/** + * Binary stream writer + */ + +class BinaryWriter +{ +public: + inline BinaryWriter(OutStream& Stream) : m_Stream(&Stream) {} + ~BinaryWriter() = default; + + inline void Write(const void* DataPtr, size_t ByteCount) + { + m_Stream->Write(DataPtr, ByteCount, m_Offset); + m_Offset += ByteCount; + } + + uint64_t CurrentOffset() const { return m_Offset; } + +private: + RefPtr<OutStream> m_Stream; + uint64_t m_Offset = 0; +}; + +inline BinaryWriter& +operator<<(BinaryWriter& Writer, bool Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, int8_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, int16_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, int32_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, int64_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, uint8_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, uint16_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, uint32_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} +inline BinaryWriter& +operator<<(BinaryWriter& Writer, uint64_t Value) +{ + Writer.Write(&Value, sizeof Value); + return Writer; +} + +/** + * Binary stream reader + */ + +class BinaryReader +{ +public: + inline BinaryReader(InStream& Stream) : m_Stream(&Stream) {} + ~BinaryReader() = default; + + inline void Read(void* DataPtr, size_t ByteCount) + { + m_Stream->Read(DataPtr, ByteCount, m_Offset); + m_Offset += ByteCount; + } + + void Seek(uint64_t Offset) + { + ZEN_ASSERT(Offset <= m_Stream->Size()); + m_Offset = Offset; + } + + void Skip(uint64_t SkipOffset) + { + ZEN_ASSERT((m_Offset + SkipOffset) <= m_Stream->Size()); + m_Offset += SkipOffset; + } + + inline uint64_t CurrentOffset() const { return m_Offset; } + inline uint64_t AvailableBytes() const { return m_Stream->Size() - m_Offset; } + +private: + RefPtr<InStream> m_Stream; + uint64_t m_Offset = 0; +}; + +inline BinaryReader& +operator>>(BinaryReader& Reader, bool& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, int8_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, int16_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, int32_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, int64_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, uint8_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, uint16_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, uint32_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} +inline BinaryReader& +operator>>(BinaryReader& Reader, uint64_t& Value) +{ + Reader.Read(&Value, sizeof Value); + return Reader; +} + +/** + * Text stream writer + */ + +class TextWriter +{ +public: + ZENCORE_API TextWriter(OutStream& Stream); + ZENCORE_API ~TextWriter(); + + ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount); + ZENCORE_API void Writef(const char* FormatString, ...); + + inline uint64_t CurrentOffset() const { return m_CurrentOffset; } + +private: + RefPtr<OutStream> m_Stream; + uint64_t m_CurrentOffset = 0; +}; + +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const char* Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const std::string_view& Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, bool Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int8_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int16_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int32_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int64_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint8_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint16_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint32_t Value); +ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint64_t Value); + +class IndentTextWriter : public TextWriter +{ +public: + ZENCORE_API IndentTextWriter(OutStream& stream); + ZENCORE_API ~IndentTextWriter(); + + ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount) override; + + inline void Indent(int Amount) { m_IndentAmount += Amount; } + + struct Scope + { + Scope(IndentTextWriter& Outer, int IndentAmount = 2) : m_Outer(Outer), m_IndentAmount(IndentAmount) + { + m_Outer.Indent(IndentAmount); + } + + ~Scope() { m_Outer.Indent(-m_IndentAmount); } + + private: + IndentTextWriter& m_Outer; + int m_IndentAmount; + }; + +private: + int m_IndentAmount = 0; + int m_LineCursor = 0; + char m_LineBuffer[2048]; +}; + +void stream_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/streamutil.h b/zencore/include/zencore/streamutil.h new file mode 100644 index 000000000..190cd18eb --- /dev/null +++ b/zencore/include/zencore/streamutil.h @@ -0,0 +1,118 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <fmt/format.h> +#include <zencore/string.h> +#include <string> +#include <string_view> + +#include "blake3.h" +#include "iohash.h" +#include "sha1.h" +#include "stream.h" + +namespace zen { + +ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::string_view& value); +ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::string& value); + +ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::wstring_view& value); +ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::wstring& value); +ZENCORE_API TextWriter& operator<<(TextWriter& writer, const std::wstring_view& value); + +inline BinaryWriter& +operator<<(BinaryWriter& writer, const SHA1& value) +{ + writer.Write(value.Hash, sizeof value.Hash); + return writer; +} +inline BinaryReader& +operator>>(BinaryReader& reader, SHA1& value) +{ + reader.Read(value.Hash, sizeof value.Hash); + return reader; +} +ZENCORE_API TextWriter& operator<<(TextWriter& writer, const zen::SHA1& value); + +inline BinaryWriter& +operator<<(BinaryWriter& writer, const BLAKE3& value) +{ + writer.Write(value.Hash, sizeof value.Hash); + return writer; +} +inline BinaryReader& +operator>>(BinaryReader& reader, BLAKE3& value) +{ + reader.Read(value.Hash, sizeof value.Hash); + return reader; +} +ZENCORE_API TextWriter& operator<<(TextWriter& writer, const BLAKE3& value); + +inline BinaryWriter& +operator<<(BinaryWriter& writer, const IoHash& value) +{ + writer.Write(value.Hash, sizeof value.Hash); + return writer; +} +inline BinaryReader& +operator>>(BinaryReader& reader, IoHash& value) +{ + reader.Read(value.Hash, sizeof value.Hash); + return reader; +} +ZENCORE_API TextWriter& operator<<(TextWriter& writer, const IoHash& value); + +} // namespace zen + +////////////////////////////////////////////////////////////////////////// + +template<> +struct fmt::formatter<zen::IoHash> +{ + constexpr auto parse(format_parse_context& ctx) + { + // Parse the presentation format and store it in the formatter: + auto it = ctx.begin(), end = ctx.end(); + + // Check if reached the end of the range: + if (it != end && *it != '}') + throw format_error("invalid format"); + + // Return an iterator past the end of the parsed range: + return it; + } + + template<typename FormatContext> + auto format(const zen::IoHash& h, FormatContext& ctx) + { + zen::ExtendableStringBuilder<48> String; + h.ToHexString(String); + return format_to(ctx.out(), std::string_view(String)); + } +}; + +template<> +struct fmt::formatter<zen::BLAKE3> +{ + constexpr auto parse(format_parse_context& ctx) + { + // Parse the presentation format and store it in the formatter: + auto it = ctx.begin(), end = ctx.end(); + + // Check if reached the end of the range: + if (it != end && *it != '}') + throw format_error("invalid format"); + + // Return an iterator past the end of the parsed range: + return it; + } + + template<typename FormatContext> + auto format(const zen::BLAKE3& h, FormatContext& ctx) + { + zen::ExtendableStringBuilder<80> String; + h.ToHexString(String); + return format_to(ctx.out(), std::string_view(String)); + } +}; diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h new file mode 100644 index 000000000..d7727ca08 --- /dev/null +++ b/zencore/include/zencore/string.h @@ -0,0 +1,595 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "intmath.h" +#include "zencore.h" + +#include <stdint.h> +#include <string.h> +#include <charconv> +#include <codecvt> +#include <concepts> +#include <optional> +#include <span> +#include <string_view> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +inline bool +StringEquals(const char8_t* s1, const char* s2) +{ + return strcmp(reinterpret_cast<const char*>(s1), s2) == 0; +} + +inline bool +StringEquals(const char* s1, const char* s2) +{ + return strcmp(s1, s2) == 0; +} + +inline size_t +StringLength(const char* str) +{ + return strlen(str); +} + +inline bool +StringEquals(const wchar_t* s1, const wchar_t* s2) +{ + return wcscmp(s1, s2) == 0; +} + +inline size_t +StringLength(const wchar_t* str) +{ + return wcslen(str); +} + +////////////////////////////////////////////////////////////////////////// +// File name helpers +// + +ZENCORE_API const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr); + +////////////////////////////////////////////////////////////////////////// +// Text formatting of numbers +// + +ZENCORE_API bool ToString(std::span<char> Buffer, uint64_t Num); +ZENCORE_API bool ToString(std::span<char> Buffer, int64_t Num); + +struct TextNumBase +{ + inline const char* c_str() const { return m_Buffer; } + inline operator std::string_view() const { return std::string_view(m_Buffer); } + +protected: + char m_Buffer[24]; +}; + +struct IntNum : public TextNumBase +{ + inline IntNum(std::unsigned_integral auto Number) { ToString(m_Buffer, uint64_t(Number)); } + inline IntNum(std::signed_integral auto Number) { ToString(m_Buffer, int64_t(Number)); } +}; + +////////////////////////////////////////////////////////////////////////// +// +// Quick-and-dirty string builder. Good enough for me, but contains traps +// and not-quite-ideal behaviour especially when mixing character types etc +// + +template<typename C> +class StringBuilderImpl +{ +public: + StringBuilderImpl() = default; + ZENCORE_API ~StringBuilderImpl(); + + StringBuilderImpl(const StringBuilderImpl&) = delete; + StringBuilderImpl(const StringBuilderImpl&&) = delete; + const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete; + const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete; + + StringBuilderImpl& Append(C OneChar) + { + EnsureCapacity(1); + + *m_CurPos++ = OneChar; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const std::string_view& String) + { + const size_t len = String.size(); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = String[i]; + + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const std::u8string_view& String) + { + const size_t len = String.size(); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = String[i]; + + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendAscii(const char* NulTerminatedString) + { + size_t StringLen = StringLength(NulTerminatedString); + + return AppendAscii({NulTerminatedString, StringLen}); + } + + inline StringBuilderImpl& Append(const char8_t* NulTerminatedString) + { + // This is super hacky and not fully functional - needs better + // solution + if constexpr (sizeof(C) == 1) + { + size_t len = StringLength((const char*)NulTerminatedString); + + EnsureCapacity(len); + + for (size_t i = 0; i < len; ++i) + m_CurPos[i] = C(NulTerminatedString[i]); + + m_CurPos += len; + } + else + { + ZEN_NOT_IMPLEMENTED(); + } + + return *this; + } + + inline StringBuilderImpl& AppendAsciiRange(const char* BeginString, const char* EndString) + { + EnsureCapacity(EndString - BeginString); + + while (BeginString != EndString) + *m_CurPos++ = *BeginString++; + + return *this; + } + + inline StringBuilderImpl& Append(const C* NulTerminatedString) + { + size_t Len = StringLength(NulTerminatedString); + + EnsureCapacity(Len); + memcpy(m_CurPos, NulTerminatedString, Len * sizeof(C)); + m_CurPos += Len; + + return *this; + } + + inline StringBuilderImpl& Append(const C* NulTerminatedString, size_t MaxChars) + { + size_t len = Min(MaxChars, StringLength(NulTerminatedString)); + + EnsureCapacity(len); + memcpy(m_CurPos, NulTerminatedString, len * sizeof(C)); + m_CurPos += len; + + return *this; + } + + inline StringBuilderImpl& AppendRange(const C* BeginString, const C* EndString) + { + size_t Len = EndString - BeginString; + + EnsureCapacity(Len); + memcpy(m_CurPos, BeginString, Len * sizeof(C)); + m_CurPos += Len; + + return *this; + } + + inline StringBuilderImpl& Append(const std::basic_string_view<C>& String) + { + return AppendRange(String.data(), String.data() + String.size()); + } + + inline const C* c_str() const + { + EnsureNulTerminated(); + return m_Base; + } + + inline C* Data() + { + EnsureNulTerminated(); + return m_Base; + } + + inline const C* Data() const + { + EnsureNulTerminated(); + return m_Base; + } + + inline size_t Size() const { return m_CurPos - m_Base; } + inline bool IsDynamic() const { return m_IsDynamic; } + inline void Reset() { m_CurPos = m_Base; } + + inline StringBuilderImpl& operator<<(uint64_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int64_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint32_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int32_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint16_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int16_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(uint8_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + inline StringBuilderImpl& operator<<(int8_t n) + { + IntNum Str(n); + return AppendAscii(Str); + } + + inline StringBuilderImpl& operator<<(const char* str) { return AppendAscii(str); } + inline StringBuilderImpl& operator<<(const std::string_view str) { return AppendAscii(str); } + inline StringBuilderImpl& operator<<(const std::u8string_view str) { return AppendAscii(str); } + inline StringBuilderImpl& operator<<(bool v) + { + using namespace std::literals; + if (v) + { + return AppendAscii("true"sv); + } + return AppendAscii("false"sv); + } + +protected: + inline void Init(C* Base, size_t Capacity) + { + m_Base = m_CurPos = Base; + m_End = Base + Capacity; + } + + inline void EnsureNulTerminated() const { *m_CurPos = '\0'; } + + inline void EnsureCapacity(size_t ExtraRequired) + { + // precondition: we know the current buffer has enough capacity + // for the existing string including NUL terminator + + if ((m_CurPos + ExtraRequired) < m_End) + return; + + Extend(ExtraRequired); + } + + ZENCORE_API void Extend(size_t ExtraCapacity); + ZENCORE_API void* AllocBuffer(size_t ByteCount); + ZENCORE_API void FreeBuffer(void* Buffer, size_t ByteCount); + + ZENCORE_API [[noreturn]] void Fail(const char* FailReason); // note: throws exception + + C* m_Base; + C* m_CurPos; + C* m_End; + bool m_IsDynamic = false; + bool m_IsExtendable = false; +}; + +////////////////////////////////////////////////////////////////////////// + +extern template StringBuilderImpl<char>; + +class StringBuilderBase : public StringBuilderImpl<char> +{ +public: + inline StringBuilderBase(char* bufferPointer, size_t bufferCapacity) { Init(bufferPointer, bufferCapacity); } + inline ~StringBuilderBase() = default; + + // Note that we don't need a terminator for the string_view so we avoid calling data() here + inline operator std::string_view() const { return std::string_view(m_Base, m_CurPos - m_Base); } + inline std::string_view ToView() const { return std::string_view(m_Base, m_CurPos - m_Base); } + inline std::string ToString() const { return std::string{Data(), Size()}; } + + inline void AppendCodepoint(uint32_t cp) + { + if (cp < 0x80) // one octet + { + Append(static_cast<char8_t>(cp)); + } + else if (cp < 0x800) + { + EnsureCapacity(2); // two octets + m_CurPos[0] = static_cast<char8_t>((cp >> 6) | 0xc0); + m_CurPos[1] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 2; + } + else if (cp < 0x10000) + { + EnsureCapacity(3); // three octets + m_CurPos[0] = static_cast<char8_t>((cp >> 12) | 0xe0); + m_CurPos[1] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80); + m_CurPos[2] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 3; + } + else + { + EnsureCapacity(4); // four octets + m_CurPos[0] = static_cast<char8_t>((cp >> 18) | 0xf0); + m_CurPos[1] = static_cast<char8_t>(((cp >> 12) & 0x3f) | 0x80); + m_CurPos[2] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80); + m_CurPos[3] = static_cast<char8_t>((cp & 0x3f) | 0x80); + m_CurPos += 4; + } + } +}; + +template<size_t N> +class StringBuilder : public StringBuilderBase +{ +public: + inline StringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) {} + inline ~StringBuilder() = default; + +private: + char m_StringBuffer[N]; +}; + +template<size_t N> +class ExtendableStringBuilder : public StringBuilderBase +{ +public: + inline ExtendableStringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) { m_IsExtendable = true; } + inline ~ExtendableStringBuilder() = default; + +private: + char m_StringBuffer[N]; +}; + +////////////////////////////////////////////////////////////////////////// + +extern template StringBuilderImpl<wchar_t>; + +class WideStringBuilderBase : public StringBuilderImpl<wchar_t> +{ +public: + inline WideStringBuilderBase(wchar_t* BufferPointer, size_t BufferCapacity) { Init(BufferPointer, BufferCapacity); } + inline ~WideStringBuilderBase() = default; + + inline operator std::wstring_view() const { return std::wstring_view{Data(), Size()}; } + inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; } + inline std::wstring toString() const { return std::wstring{Data(), Size()}; } + + inline StringBuilderImpl& operator<<(const std::u16string_view str) { return Append((const wchar_t*)str.data(), str.size()); } + inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); } + using StringBuilderImpl:: operator<<; +}; + +template<size_t N> +class WideStringBuilder : public WideStringBuilderBase +{ +public: + inline WideStringBuilder() : WideStringBuilderBase(m_Buffer, N) {} + ~WideStringBuilder() = default; + +private: + wchar_t m_Buffer[N]; +}; + +template<size_t N> +class ExtendableWideStringBuilder : public WideStringBuilderBase +{ +public: + inline ExtendableWideStringBuilder() : WideStringBuilderBase(m_Buffer, N) { m_IsExtendable = true; } + ~ExtendableWideStringBuilder() = default; + +private: + wchar_t m_Buffer[N]; +}; + +////////////////////////////////////////////////////////////////////////// + +void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out); +void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out); +void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out); +std::wstring Utf8ToWide(const std::string_view& wstr); + +void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out); +std::string WideToUtf8(const wchar_t* wstr); +void WideToUtf8(const std::u16string_view& wstr, StringBuilderBase& out); +void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out); +std::string WideToUtf8(const std::wstring_view Wstr); + +/// <summary> +/// Parse hex string into a byte buffer +/// </summary> +/// <param name="string">Input string</param> +/// <param name="characterCount">Number of characters in string</param> +/// <param name="outPtr">Pointer to output buffer</param> +/// <returns>true if the input consisted of all valid hexadecimal characters</returns> + +inline bool +ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr) +{ + ZEN_ASSERT((CharacterCount & 1) == 0); + + auto char2nibble = [](char c) { + uint8_t c8 = uint8_t(c - '0'); + + if (c8 < 10) + return c8; + + c8 -= 'A' - '0' - 10; + + if (c8 < 16) + return c8; + + c8 -= 'a' - 'A'; + + if (c8 < 16) + return c8; + + return uint8_t(0xff); + }; + + uint8_t allBits = 0; + + while (CharacterCount) + { + uint8_t n0 = char2nibble(InputString[0]); + uint8_t n1 = char2nibble(InputString[1]); + + allBits |= n0 | n1; + + *OutPtr = (n0 << 4) | n1; + + OutPtr += 1; + InputString += 2; + CharacterCount -= 2; + } + + return (allBits & 0x80) == 0; +} + +inline void +ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString) +{ + const char hexchars[] = "0123456789abcdef"; + + while (ByteCount--) + { + uint8_t byte = *InputData++; + + *OutString++ = hexchars[byte >> 4]; + *OutString++ = hexchars[byte & 15]; + } +} + +////////////////////////////////////////////////////////////////////////// +// Format numbers for humans +// + +ZENCORE_API size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer); +ZENCORE_API size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer); +ZENCORE_API size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer); +ZENCORE_API size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer); +ZENCORE_API size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer); + +struct NiceBase +{ + inline const char* c_str() const { return m_Buffer; } + inline operator std::string_view() const { return std::string_view(m_Buffer); } + +protected: + char m_Buffer[16]; +}; + +struct NiceNum : public NiceBase +{ + inline NiceNum(uint64_t Num) { NiceNumToBuffer(Num, m_Buffer); } +}; + +struct NiceBytes : public NiceBase +{ + inline NiceBytes(uint64_t Num) { NiceBytesToBuffer(Num, m_Buffer); } +}; + +struct NiceByteRate : public NiceBase +{ + inline NiceByteRate(uint64_t Bytes, uint64_t TimeMilliseconds) { NiceByteRateToBuffer(Bytes, TimeMilliseconds, m_Buffer); } +}; + +struct NiceLatencyNs : public NiceBase +{ + inline NiceLatencyNs(uint64_t Milliseconds) { NiceLatencyNsToBuffer(Milliseconds, m_Buffer); } +}; + +struct NiceTimeSpanMs : public NiceBase +{ + inline NiceTimeSpanMs(uint64_t Milliseconds) { NiceTimeSpanMsToBuffer(Milliseconds, m_Buffer); } +}; + +////////////////////////////////////////////////////////////////////////// + +inline std::string +NiceRate(uint64_t Num, uint32_t DurationMilliseconds, const char* Unit = "B") +{ + char Buffer[32]; + + if (DurationMilliseconds) + { + NiceNumToBuffer(Num * 1000 / DurationMilliseconds, Buffer); + } + else + { + strcpy_s(Buffer, "0"); + } + + strcat_s(Buffer, Unit); + strcat_s(Buffer, "/s"); + + return Buffer; +} + +////////////////////////////////////////////////////////////////////////// + +template<std::integral T> +std::optional<T> +ParseInt(const std::string_view& Input) +{ + T Out; + const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out); + if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range) + { + return std::nullopt; + } + return Out; +} + +////////////////////////////////////////////////////////////////////////// + +void string_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h new file mode 100644 index 000000000..48afad33f --- /dev/null +++ b/zencore/include/zencore/thread.h @@ -0,0 +1,118 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +namespace zen { + +/** + * Reader-writer lock + * + * - A single thread may hold an exclusive lock at any given moment + * + * - Multiple threads may hold shared locks, but only if no thread has + * acquired an exclusive lock + */ +class RwLock +{ +public: + ZENCORE_API void AcquireShared(); + ZENCORE_API void ReleaseShared(); + + ZENCORE_API void AcquireExclusive(); + ZENCORE_API void ReleaseExclusive(); + + struct SharedLockScope + { + SharedLockScope(RwLock& lock) : m_Lock(lock) { m_Lock.AcquireShared(); } + ~SharedLockScope() { m_Lock.ReleaseShared(); } + + private: + RwLock& m_Lock; + }; + + struct ExclusiveLockScope + { + ExclusiveLockScope(RwLock& lock) : m_Lock(lock) { m_Lock.AcquireExclusive(); } + ~ExclusiveLockScope() { m_Lock.ReleaseExclusive(); } + + private: + RwLock& m_Lock; + }; + +private: + void* m_Srw = nullptr; +}; + +/** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore') + */ +class Event +{ +public: + ZENCORE_API Event(); + ZENCORE_API ~Event(); + + Event(Event&& Rhs) : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; } + + Event(const Event& Rhs) = delete; + Event& operator=(const Event& Rhs) = delete; + + inline Event& operator=(Event&& Rhs) + { + m_EventHandle = Rhs.m_EventHandle; + Rhs.m_EventHandle = nullptr; + return *this; + } + + ZENCORE_API void Set(); + ZENCORE_API void Reset(); + ZENCORE_API bool Wait(int TimeoutMs = -1); + +protected: + explicit Event(void* EventHandle) : m_EventHandle(EventHandle) {} + + void* m_EventHandle = nullptr; +}; + +/** Basic abstraction of an IPC mechanism (aka 'binary semaphore') + */ +class NamedEvent : public Event +{ +public: + ZENCORE_API explicit NamedEvent(std::string_view EventName); + ZENCORE_API explicit NamedEvent(std::u8string_view EventName); +}; + +/** Basic process abstraction + */ +class Process +{ +public: + ZENCORE_API Process(); + + Process(const Process&) = delete; + Process& operator=(const Process&) = delete; + + ZENCORE_API ~Process(); + + ZENCORE_API void Initialize(int Pid); + ZENCORE_API void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle + ZENCORE_API bool IsRunning() const; + ZENCORE_API bool IsValid() const; + ZENCORE_API bool Wait(int TimeoutMs = -1); + ZENCORE_API void Terminate(int ExitCode); + inline int Pid() const { return m_Pid; } + +private: + void* m_ProcessHandle = nullptr; + int m_Pid = 0; +}; + +ZENCORE_API bool IsProcessRunning(int pid); + +ZENCORE_API void Sleep(int ms); + +void thread_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/timer.h b/zencore/include/zencore/timer.h new file mode 100644 index 000000000..c9122eb44 --- /dev/null +++ b/zencore/include/zencore/timer.h @@ -0,0 +1,41 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <intrin.h> +#include <stdint.h> +#include "zencore.h" + +namespace zen { + +// High frequency timers + +ZENCORE_API uint64_t GetHifreqTimerValue(); +ZENCORE_API uint64_t GetHifreqTimerFrequency(); +ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init + +class Stopwatch +{ +public: + Stopwatch() : m_StartValue(GetHifreqTimerValue()) {} + + inline uint64_t getElapsedTimeMs() { return (GetHifreqTimerValue() - m_StartValue) * 1000 / GetHifreqTimerFrequency(); } + + inline void reset() { m_StartValue = GetHifreqTimerValue(); } + +private: + uint64_t m_StartValue; +}; + +// CPU timers + +inline uint64_t +GetCpuTimerValue() +{ + unsigned int foo; + return __rdtscp(&foo); +} + +void timer_forcelink(); // internal + +} // namespace zen diff --git a/zencore/include/zencore/trace.h b/zencore/include/zencore/trace.h new file mode 100644 index 000000000..191ce4a3a --- /dev/null +++ b/zencore/include/zencore/trace.h @@ -0,0 +1,91 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <inttypes.h> +#include "zencore.h" + +#pragma section("trace_events", read) +#define U_TRACE_DECL __declspec(allocate("trace_events")) + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +struct TraceSite +{ + const char* sourceFile; + const uint32_t sourceLine; + const uint32_t flags; +}; + +struct TraceEvent +{ + const TraceSite* site; + ThreadId_t threadId; + const char* message; +}; + +enum TraceFlags +{ + kTrace_Debug = 1 << 0, + kTrace_Info = 1 << 1, + kTrace_Warn = 1 << 2, + kTrace_Error = 1 << 3, + kTrace_Fatal = 1 << 4, + + kTrace_Trace = 1 << 7, +}; + +class Tracer +{ +public: + void Log(const TraceEvent& e); + + __forceinline uint32_t Accept(const TraceSite& e) const { return (m_acceptFlags & e.flags); } + +private: + uint32_t m_acceptFlags = ~0u; +}; + +ZENCORE_API extern Tracer g_globalTracer; + +/** Trace event handler + */ +class TraceHandler +{ +public: + virtual void Trace(const TraceEvent& e) = 0; + +private: +}; + +ZENCORE_API static void TraceBroadcast(const TraceEvent& e); + +void trace_forcelink(); // internal + +} // namespace zen + +__forceinline zen::Tracer& +CurrentTracer() +{ + return zen::g_globalTracer; +} + +#define U_LOG_GENERIC(msg, flags) \ + do \ + { \ + zen::Tracer& t = CurrentTracer(); \ + static U_TRACE_DECL constexpr zen::TraceSite traceSite{__FILE__, __LINE__, flags}; \ + const zen::TraceEvent traceEvent = {&traceSite, 0u, msg}; \ + if (t.Accept(traceSite)) \ + t.Log(traceEvent); \ + } while (false) + +////////////////////////////////////////////////////////////////////////// + +#define U_LOG_DEBUG(msg) U_LOG_GENERIC(msg, zen::kTrace_Debug) +#define U_LOG_INFO(msg) U_LOG_GENERIC(msg, zen::kTrace_Info) +#define U_LOG_WARN(msg) U_LOG_GENERIC(msg, zen::kTrace_Warn) +#define U_LOG_ERROR(msg) U_LOG_GENERIC(msg, zen::kTrace_Error) +#define U_LOG_FATAL(msg) U_LOG_GENERIC(msg, zen::kTrace_Fatal) diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h new file mode 100644 index 000000000..a793b160a --- /dev/null +++ b/zencore/include/zencore/uid.h @@ -0,0 +1,78 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> +#include <compare> + +namespace zen { + +class StringBuilderBase; + +/** Object identifier + + Can be used as a GUID essentially, but is more compact (12 bytes) and as such + is more susceptible to collisions than a 16-byte GUID but also I don't expect + the population to be large so in practice the risk should be minimal due to + how the identifiers work. + + Similar in spirit to MongoDB ObjectId + + When serialized, object identifiers generated in a given session in sequence + will sort in chronological order since the timestamp is in the MSB in big + endian format. This makes it suitable as a database key since most indexing + structures work better when keys are inserted in lexicographically + increasing order. + + The current layout is basically: + + |----------------|----------------|----------------| + | timestamp | serial # | run id | + |----------------|----------------|----------------| + MSB LSB + + - Timestamp is a unsigned 32-bit value (seconds since Jan 1 1970) + - Serial # is another unsigned 32-bit value which is assigned a (strong) + random number at initialization time which is incremented when a new Oid + is generated + - The run id is generated from a strong random number generator + at initialization time and stays fixed for the duration of the program + + Timestamp and serial are stored in memory in such a way that they can be + ordered lexicographically. I.e they are in big-endian byte order. + + */ + +struct Oid +{ + static const int StringLength = 24; + typedef char String_t[StringLength + 1]; + + static void Initialize(); + [[nodiscard]] static Oid NewOid(); + + const Oid& Generate(); + [[nodiscard]] static Oid FromHexString(const std::string_view String); + StringBuilderBase& ToString(StringBuilderBase& OutString) const; + + auto operator<=>(const Oid& rhs) const = default; + + static const Oid Zero; // Min (can be used to signify a "null" value, or for open range queries) + static const Oid Max; // Max (can be used for open range queries) + + struct Hasher + { + size_t operator()(const Oid& id) const + { + const size_t seed = id.OidBits[0]; + return (seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1]) | (uint64_t(id.OidBits[2]) << 32); + } + }; + + // You should not assume anything about these words + uint32_t OidBits[3]; +}; + +extern void uid_forcelink(); + +} // namespace zen diff --git a/zencore/include/zencore/varint.h b/zencore/include/zencore/varint.h new file mode 100644 index 000000000..0c40dd66b --- /dev/null +++ b/zencore/include/zencore/varint.h @@ -0,0 +1,255 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "intmath.h" + +namespace zen { + +// Variable-Length Integer Encoding +// +// ZigZag encoding is used to convert signed integers into unsigned integers in a way that allows +// integers with a small magnitude to have a smaller encoded representation. +// +// An unsigned integer is encoded into 1-9 bytes based on its magnitude. The first byte indicates +// how many additional bytes are used by the number of leading 1-bits that it has. The additional +// bytes are stored in big endian order, and the most significant bits of the value are stored in +// the remaining bits in the first byte. The encoding of the first byte allows the reader to skip +// over the encoded integer without consuming its bytes individually. +// +// Encoded unsigned integers sort the same in a byte-wise comparison as when their decoded values +// are compared. The same property does not hold for signed integers due to ZigZag encoding. +// +// 32-bit inputs encode to 1-5 bytes. +// 64-bit inputs encode to 1-9 bytes. +// +// 0x0000'0000'0000'0000 - 0x0000'0000'0000'007f : 0b0_______ 1 byte +// 0x0000'0000'0000'0080 - 0x0000'0000'0000'3fff : 0b10______ 2 bytes +// 0x0000'0000'0000'4000 - 0x0000'0000'001f'ffff : 0b110_____ 3 bytes +// 0x0000'0000'0020'0000 - 0x0000'0000'0fff'ffff : 0b1110____ 4 bytes +// 0x0000'0000'1000'0000 - 0x0000'0007'ffff'ffff : 0b11110___ 5 bytes +// 0x0000'0008'0000'0000 - 0x0000'03ff'ffff'ffff : 0b111110__ 6 bytes +// 0x0000'0400'0000'0000 - 0x0001'ffff'ffff'ffff : 0b1111110_ 7 bytes +// 0x0002'0000'0000'0000 - 0x00ff'ffff'ffff'ffff : 0b11111110 8 bytes +// 0x0100'0000'0000'0000 - 0xffff'ffff'ffff'ffff : 0b11111111 9 bytes +// +// Encoding Examples +// -42 => ZigZag => 0x53 => 0x53 +// 42 => ZigZag => 0x54 => 0x54 +// 0x1 => 0x01 +// 0x12 => 0x12 +// 0x123 => 0x81 0x23 +// 0x1234 => 0x92 0x34 +// 0x12345 => 0xc1 0x23 0x45 +// 0x123456 => 0xd2 0x34 0x56 +// 0x1234567 => 0xe1 0x23 0x45 0x67 +// 0x12345678 => 0xf0 0x12 0x34 0x56 0x78 +// 0x123456789 => 0xf1 0x23 0x45 0x67 0x89 +// 0x123456789a => 0xf8 0x12 0x34 0x56 0x78 0x9a +// 0x123456789ab => 0xfb 0x23 0x45 0x67 0x89 0xab +// 0x123456789abc => 0xfc 0x12 0x34 0x56 0x78 0x9a 0xbc +// 0x123456789abcd => 0xfd 0x23 0x45 0x67 0x89 0xab 0xcd +// 0x123456789abcde => 0xfe 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde +// 0x123456789abcdef => 0xff 0x01 0x23 0x45 0x67 0x89 0xab 0xcd 0xef +// 0x123456789abcdef0 => 0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0 + +/** + * Measure the length in bytes (1-9) of an encoded variable-length integer. + * + * @param InData A variable-length encoding of an (signed or unsigned) integer. + * @return The number of bytes used to encode the integer, in the range 1-9. + */ +inline uint32_t +MeasureVarUInt(const void* InData) +{ + return CountLeadingZeros(uint8_t(~*static_cast<const uint8_t*>(InData))) - 23; +} + +/** Measure the length in bytes (1-9) of an encoded variable-length integer. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(const void* InData) +{ + return MeasureVarUInt(InData); +} + +/** Measure the number of bytes (1-5) required to encode the 32-bit input. */ +inline uint32_t +MeasureVarUInt(uint32_t InValue) +{ + return uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1); +} + +/** Measure the number of bytes (1-9) required to encode the 64-bit input. */ +inline uint32_t +MeasureVarUInt(uint64_t InValue) +{ + return uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9)); +} + +/** Measure the number of bytes (1-5) required to encode the 32-bit input. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(int32_t InValue) +{ + return MeasureVarUInt(uint32_t((InValue >> 31) ^ (InValue << 1))); +} + +/** Measure the number of bytes (1-9) required to encode the 64-bit input. \see \ref MeasureVarUInt */ +inline uint32_t +MeasureVarInt(int64_t InValue) +{ + return MeasureVarUInt(uint64_t((InValue >> 63) ^ (InValue << 1))); +} + +/** + * Read a variable-length unsigned integer. + * + * @param InData A variable-length encoding of an unsigned integer. + * @param OutByteCount The number of bytes consumed from the input. + * @return An unsigned integer. + */ +inline uint64_t +ReadVarUInt(const void* InData, uint32_t& OutByteCount) +{ + const uint32_t ByteCount = MeasureVarUInt(InData); + OutByteCount = ByteCount; + + const uint8_t* InBytes = static_cast<const uint8_t*>(InData); + uint64_t Value = *InBytes++ & uint8_t(0xff >> ByteCount); + switch (ByteCount - 1) + { + case 8: + Value <<= 8; + Value |= *InBytes++; + case 7: + Value <<= 8; + Value |= *InBytes++; + case 6: + Value <<= 8; + Value |= *InBytes++; + case 5: + Value <<= 8; + Value |= *InBytes++; + case 4: + Value <<= 8; + Value |= *InBytes++; + case 3: + Value <<= 8; + Value |= *InBytes++; + case 2: + Value <<= 8; + Value |= *InBytes++; + case 1: + Value <<= 8; + Value |= *InBytes++; + default: + return Value; + } +} + +/** + * Read a variable-length signed integer. + * + * @param InData A variable-length encoding of a signed integer. + * @param OutByteCount The number of bytes consumed from the input. + * @return A signed integer. + */ +inline int64_t +ReadVarInt(const void* InData, uint32_t& OutByteCount) +{ + const uint64_t Value = ReadVarUInt(InData, OutByteCount); + return -int64_t(Value & 1) ^ int64_t(Value >> 1); +} + +/** + * Write a variable-length unsigned integer. + * + * @param InValue An unsigned integer to encode. + * @param OutData A buffer of at least 5 bytes to write the output to. + * @return The number of bytes used in the output. + */ +inline uint32_t +WriteVarUInt(uint32_t InValue, void* OutData) +{ + const uint32_t ByteCount = MeasureVarUInt(InValue); + uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1; + switch (ByteCount - 1) + { + case 4: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 3: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 2: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 1: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + default: + break; + } + *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue); + return ByteCount; +} + +/** + * Write a variable-length unsigned integer. + * + * @param InValue An unsigned integer to encode. + * @param OutData A buffer of at least 9 bytes to write the output to. + * @return The number of bytes used in the output. + */ +inline uint32_t +WriteVarUInt(uint64_t InValue, void* OutData) +{ + const uint32_t ByteCount = MeasureVarUInt(InValue); + uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1; + switch (ByteCount - 1) + { + case 8: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 7: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 6: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 5: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 4: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 3: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 2: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + case 1: + *OutBytes-- = uint8_t(InValue); + InValue >>= 8; + default: + break; + } + *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue); + return ByteCount; +} + +/** Write a variable-length signed integer. \see \ref WriteVarUInt */ +inline uint32_t +WriteVarInt(int32_t InValue, void* OutData) +{ + const uint32_t Value = uint32_t((InValue >> 31) ^ (InValue << 1)); + return WriteVarUInt(Value, OutData); +} + +/** Write a variable-length signed integer. \see \ref WriteVarUInt */ +inline uint32_t +WriteVarInt(int64_t InValue, void* OutData) +{ + const uint64_t Value = uint64_t((InValue >> 63) ^ (InValue << 1)); + return WriteVarUInt(Value, OutData); +} + +} // namespace zen diff --git a/zencore/include/zencore/windows.h b/zencore/include/zencore/windows.h new file mode 100644 index 000000000..8888bf757 --- /dev/null +++ b/zencore/include/zencore/windows.h @@ -0,0 +1,10 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- +#ifndef NOMINMAX +# define NOMINMAX // We don't want your min/max macros +#endif +#define WIN32_LEAN_AND_MEAN +#include <windows.h> diff --git a/zencore/include/zencore/xxhash.h b/zencore/include/zencore/xxhash.h new file mode 100644 index 000000000..5407755df --- /dev/null +++ b/zencore/include/zencore/xxhash.h @@ -0,0 +1,87 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +#include <zencore/memory.h> + +#include <xxh3.h> +#include <string_view> + +namespace zen { + +class StringBuilderBase; + +/** + * XXH3 hash + */ +struct XXH3_128 +{ + uint8_t Hash[16]; + + static XXH3_128 MakeFrom(const void* data /* 16 bytes */) + { + XXH3_128 Xx; + memcpy(Xx.Hash, data, sizeof Xx); + return Xx; + } + + static inline XXH3_128 HashMemory(const void* data, size_t byteCount) + { + XXH3_128 Hash; + XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits(data, byteCount)); + return Hash; + } + static XXH3_128 HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); } + static XXH3_128 FromHexString(const char* string); + static XXH3_128 FromHexString(const std::string_view string); + const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const; + StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; + + static const int StringLength = 32; + typedef char String_t[StringLength + 1]; + + static XXH3_128 Zero; // Initialized to all zeros + + inline auto operator<=>(const XXH3_128& rhs) const = default; + + struct Hasher + { + size_t operator()(const XXH3_128& v) const + { + size_t h; + memcpy(&h, v.Hash, sizeof h); + return h; + } + }; +}; + +struct XXH3_128Stream +{ + /// Begin streaming hash compute (not needed on freshly constructed instance) + void Reset() { memset(&m_State, 0, sizeof m_State); } + + /// Append another chunk + XXH3_128Stream& Append(const void* Data, size_t ByteCount) + { + XXH3_128bits_update(&m_State, Data, ByteCount); + return *this; + } + + /// Append another chunk + XXH3_128Stream& Append(MemoryView Data) { return Append(Data.GetData(), Data.GetSize()); } + + /// Obtain final hash. If you wish to reuse the instance call reset() + XXH3_128 GetHash() + { + XXH3_128 Hash; + XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits_digest(&m_State)); + return Hash; + } + +private: + XXH3_state_s m_State{}; +}; + +} // namespace zen diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h new file mode 100644 index 000000000..4a448776b --- /dev/null +++ b/zencore/include/zencore/zencore.h @@ -0,0 +1,134 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <cinttypes> +#include <exception> +#include <string> + +////////////////////////////////////////////////////////////////////////// +// Platform +// + +#define ZEN_PLATFORM_WINDOWS 1 +#define ZEN_PLATFORM_LINUX 0 +#define ZEN_PLATFORM_MACOS 0 + +////////////////////////////////////////////////////////////////////////// +// Compiler +// + +#ifdef _MSC_VER +# define ZEN_COMPILER_MSC 1 +#endif + +#ifndef ZEN_COMPILER_MSC +# define ZEN_COMPILER_MSC 0 +#endif + +#ifndef ZEN_COMPILER_CLANG +# define ZEN_COMPILER_CLANG 0 +#endif + +////////////////////////////////////////////////////////////////////////// +// Build flavor +// + +#ifdef NDEBUG +# define ZEN_BUILD_DEBUG 0 +# define ZEN_BUILD_RELEASE 1 +#else +# define ZEN_BUILD_DEBUG 1 +# define ZEN_BUILD_RELEASE 0 +#endif + +////////////////////////////////////////////////////////////////////////// + +#define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1 + +////////////////////////////////////////////////////////////////////////// +// Assert +// + +namespace zen { + +class AssertException : public std::exception +{ +public: + AssertException(const char* Msg) : m_Msg(Msg) {} + + [[nodiscard]] virtual char const* what() const override { return m_Msg.c_str(); } + +private: + std::string m_Msg; +}; + +} // namespace zen + +#define ZEN_ASSERT(x, ...) \ + do \ + { \ + if (x) \ + break; \ + throw ::zen::AssertException{#x}; \ + } while (false) + +#ifndef NDEBUG +# define ZEN_ASSERT_SLOW(x, ...) \ + do \ + { \ + if (x) \ + break; \ + throw ::zen::AssertException{#x}; \ + } while (false) +#else +# define ZEN_ASSERT_SLOW(x, ...) +#endif + +////////////////////////////////////////////////////////////////////////// + +#ifdef __clang__ +template<typename T> +auto ZenArrayCountHelper(T& t) -> typename std::enable_if<__is_array(T), char (&)[sizeof(t) / sizeof(t[0]) + 1]>::Type; +#else +template<typename T, uint32_t N> +char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; +#endif + +#define ZEN_ARRAY_COUNT(array) (sizeof(ZenArrayCountHelper(array)) - 1) + +////////////////////////////////////////////////////////////////////////// + +#define ZEN_NOINLINE __declspec(noinline) +#define ZEN_UNUSED(...) ((void)__VA_ARGS__) +#define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false) +#define ZENCORE_API // Placeholder to allow DLL configs in the future + +ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is within the stack of the currently executing thread +ZENCORE_API bool IsApplicationExitRequested(); +ZENCORE_API void RequestApplicationExit(int ExitCode); + +ZENCORE_API void zencore_forcelinktests(); + +////////////////////////////////////////////////////////////////////////// + +#if ZEN_COMPILER_MSC +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on)) +#else +#endif + +// Set up optimization control macros, now that we have both the build settings and the platform macros +#define ZEN_DISABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL + +#if ZEN_BUILD_DEBUG +# define ZEN_ENABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL +#else +# define ZEN_ENABLE_OPTIMIZATION ZEN_ENABLE_OPTIMIZATION_ACTUAL +#endif + +#define ZEN_ENABLE_OPTIMIZATION_ALWAYS ZEN_ENABLE_OPTIMIZATION_ACTUAL + +////////////////////////////////////////////////////////////////////////// + +using ThreadId_t = uint32_t; diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp new file mode 100644 index 000000000..a42dd83f4 --- /dev/null +++ b/zencore/iobuffer.cpp @@ -0,0 +1,341 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/iobuffer.h> + +#include <doctest/doctest.h> +#include <memory.h> +#include <zencore/memory.h> +#include <zencore/thread.h> +#include <system_error> + +#include <atlfile.h> +#include <spdlog/spdlog.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +void* +IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment) +{ + if (((InSize & 0xffFF) == 0) && (Alignment == 0x10000)) + { + m_Flags |= kLowLevelAlloc; + return VirtualAlloc(nullptr, InSize, MEM_COMMIT, PAGE_READWRITE); + } + else + { + return Memory::Alloc(InSize, Alignment); + } +} + +void +IoBufferCore::FreeBuffer() +{ + if (m_Flags & kLowLevelAlloc) + { + VirtualFree(const_cast<void*>(m_DataPtr), 0, MEM_DECOMMIT); + } + else + { + return Memory::Free(const_cast<void*>(m_DataPtr)); + } +} + +////////////////////////////////////////////////////////////////////////// + +IoBufferCore::IoBufferCore(size_t InSize) +{ + static_assert(sizeof(IoBufferCore) == 32); + + m_DataPtr = AllocateBuffer(InSize, sizeof(void*)); + m_DataBytes = InSize; + + SetIsOwned(true); +} + +IoBufferCore::IoBufferCore(size_t InSize, size_t Alignment) +{ + m_DataPtr = AllocateBuffer(InSize, Alignment); + m_DataBytes = InSize; + + SetIsOwned(true); +} + +IoBufferCore::~IoBufferCore() +{ + if (IsOwned() && m_DataPtr) + { + FreeBuffer(); + m_DataPtr = nullptr; + } +} + +void +IoBufferCore::DeleteThis() const +{ + // We do this just to avoid paying for the cost of a vtable + if (const IoBufferExtendedCore* _ = ExtendedCore()) + { + delete _; + } + else + { + delete this; + } +} + +void +IoBufferCore::Materialize() const +{ + if (const IoBufferExtendedCore* _ = ExtendedCore()) + { + _->Materialize(); + } +} + +void +IoBufferCore::MakeOwned(bool Immutable) +{ + if (!IsOwned()) + { + void* OwnedDataPtr = AllocateBuffer(m_DataBytes, sizeof(void*)); + memcpy(OwnedDataPtr, m_DataPtr, m_DataBytes); + + m_DataPtr = OwnedDataPtr; + SetIsOwned(true); + } + + SetIsImmutable(Immutable); +} + +////////////////////////////////////////////////////////////////////////// + +IoBufferExtendedCore::IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership) +: IoBufferCore(nullptr, Size) +, m_FileHandle(FileHandle) +, m_FileOffset(Offset) +{ + m_Flags |= kIsOwned | kIsExtended; + + if (TransferHandleOwnership) + { + m_Flags |= kOwnsFile; + } +} + +IoBufferExtendedCore::IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size) +: IoBufferCore(Outer, nullptr, Size) +, m_FileHandle(Outer->m_FileHandle) +, m_FileOffset(Outer->m_FileOffset + Offset) +{ + m_Flags |= kIsOwned | kIsExtended; +} + +IoBufferExtendedCore::~IoBufferExtendedCore() +{ + if (m_MappedPointer) + { + UnmapViewOfFile(m_MappedPointer); + } + + if (m_Flags & kOwnsMmap) + { + CloseHandle(m_MmapHandle); + } + + if (m_Flags & kOwnsFile) + { + BOOL Success = CloseHandle(m_FileHandle); + + if (!Success) + { + spdlog::warn("Error reported on file handle close!"); + } + } + + m_DataPtr = nullptr; +} + +RwLock g_MappingLock; + +void +IoBufferExtendedCore::Materialize() const +{ + // The synchronization scheme here is very primitive, if we end up with + // a lot of contention we can make it more fine-grained + + if (m_MmapHandle) + return; + + RwLock::ExclusiveLockScope _(g_MappingLock); + + // Someone could have gotten here first + if (m_MmapHandle) + return; + + m_MmapHandle = CreateFileMapping(m_FileHandle, + /* lpFileMappingAttributes */ nullptr, + /* flProtect */ PAGE_READONLY, + /* dwMaximumSizeLow */ 0, + /* dwMaximumSizeHigh */ 0, + /* lpName */ nullptr); + + if (m_MmapHandle == nullptr) + { + throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file copy failed"); + } + + m_Flags |= kOwnsMmap; + + const uint64_t MapOffset = m_FileOffset & ~0xffffull; + const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset; + const uint64_t MapSize = (MappedOffsetDisplacement + m_DataBytes + 0xffffu) & ~0xffffull; + + void* MappedBase = MapViewOfFile(m_MmapHandle, + /* dwDesiredAccess */ FILE_MAP_READ, + /* FileOffsetHigh */ uint32_t(MapOffset >> 32), + /* FileOffsetLow */ uint32_t(MapOffset & 0xffFFffFFu), + /* dwNumberOfBytesToMap */ m_DataBytes); + + if (MappedBase == nullptr) + { + throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "MapViewOfFile failed"); + } + + m_MappedPointer = MappedBase; + m_DataPtr = reinterpret_cast<uint8_t*>(MappedBase) + MappedOffsetDisplacement; + + m_Flags |= kIsMaterialized; +} + +bool +IoBufferExtendedCore::GetFileReference(IoBufferFileReference& OutRef) const +{ + if (m_FileHandle == nullptr) + { + return false; + } + + OutRef.FileHandle = m_FileHandle; + OutRef.FileChunkOffset = m_FileOffset; + OutRef.FileChunkSize = m_DataBytes; + + return true; +} + +////////////////////////////////////////////////////////////////////////// + +IoBuffer::IoBuffer(size_t InSize) : m_Core(new IoBufferCore(InSize)) +{ +} + +IoBuffer::IoBuffer(size_t InSize, uint64_t InAlignment) : m_Core(new IoBufferCore(InSize, InAlignment)) +{ +} + +IoBuffer::IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t Size) +{ + if (Size == ~(0ull)) + { + Size = std::clamp<size_t>(Size, 0, OuterBuffer.Size() - Offset); + } + + ZEN_ASSERT(Offset <= OuterBuffer.Size()); + ZEN_ASSERT((Offset + Size) <= OuterBuffer.Size()); + + if (IoBufferExtendedCore* Extended = OuterBuffer.m_Core->ExtendedCore()) + { + m_Core = new IoBufferExtendedCore(Extended, Offset, Size); + } + else + { + m_Core = new IoBufferCore(OuterBuffer.m_Core, reinterpret_cast<const uint8_t*>(OuterBuffer.Data()) + Offset, Size); + } +} + +IoBuffer::IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize) +: m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ true)) +{ +} + +IoBuffer::IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize) +: m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ false)) +{ +} + +bool +IoBuffer::GetFileReference(IoBufferFileReference& OutRef) const +{ + if (IoBufferExtendedCore* ExtCore = m_Core->ExtendedCore()) + { + if (ExtCore->GetFileReference(OutRef)) + { + return true; + } + } + + // Not a file reference + + OutRef.FileHandle = 0; + OutRef.FileChunkOffset = ~0ull; + OutRef.FileChunkSize = 0; + + return false; +} + +////////////////////////////////////////////////////////////////////////// + +IoBuffer +IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size) +{ + return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); +} + +IoBuffer +IoBufferBuilder::MakeFromFile(const wchar_t* FileName, uint64_t Offset, uint64_t Size) +{ + CAtlFile DataFile; + + HRESULT hRes = DataFile.Create(FileName, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); + + if (SUCCEEDED(hRes)) + { + ULONGLONG FileSize; + DataFile.GetSize(FileSize); + + if (Size == ~0ull) + { + Size = FileSize; + } + else + { + // Clamp size + if ((Offset + Size) > FileSize) + { + Size = FileSize - Offset; + } + } + + return IoBuffer(IoBuffer::File, DataFile.Detach(), Offset, Size); + } + + return {}; +} + +////////////////////////////////////////////////////////////////////////// + +void +iobuffer_forcelink() +{ +} + +TEST_CASE("IoBuffer") +{ + zen::IoBuffer buffer1; + zen::IoBuffer buffer2(16384); + zen::IoBuffer buffer3(buffer2, 0, buffer2.Size()); +} + +} // namespace zen diff --git a/zencore/iohash.cpp b/zencore/iohash.cpp new file mode 100644 index 000000000..afe2e54ba --- /dev/null +++ b/zencore/iohash.cpp @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/iohash.h> + +#include <zencore/blake3.h> +#include <zencore/string.h> + +#include <doctest/doctest.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +IoHash IoHash::Zero; // Initialized to all zeros + +IoHash +IoHash::HashMemory(const void* data, size_t byteCount) +{ + BLAKE3 b3 = BLAKE3::HashMemory(data, byteCount); + + IoHash io; + memcpy(io.Hash, b3.Hash, sizeof io.Hash); + + return io; +} + +IoHash +IoHash::FromHexString(const char* string) +{ + return FromHexString({string, sizeof(IoHash::Hash) * 2}); +} + +IoHash +IoHash::FromHexString(std::string_view string) +{ + ZEN_ASSERT(string.size() == 2 * sizeof(IoHash::Hash)); + + IoHash io; + + ParseHexBytes(string.data(), string.size(), io.Hash); + + return io; +} + +const char* +IoHash::ToHexString(char* outString /* 40 characters + NUL terminator */) const +{ + ToHexBytes(Hash, sizeof(IoHash), outString); + outString[2 * sizeof(IoHash)] = '\0'; + + return outString; +} + +StringBuilderBase& +IoHash::ToHexString(StringBuilderBase& outBuilder) const +{ + String_t Str; + ToHexString(Str); + + outBuilder.AppendRange(Str, &Str[StringLength]); + + return outBuilder; +} + +std::string +IoHash::ToHexString() const +{ + String_t Str; + ToHexString(Str); + + return Str; +} + +} // namespace zen diff --git a/zencore/iothreadpool.cpp b/zencore/iothreadpool.cpp new file mode 100644 index 000000000..4ed81d7a2 --- /dev/null +++ b/zencore/iothreadpool.cpp @@ -0,0 +1,36 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "iothreadpool.h" + +namespace zen { + +WinIoThreadPool::WinIoThreadPool(int InThreadCount) +{ + // Thread pool setup + + m_ThreadPool = CreateThreadpool(NULL); + + SetThreadpoolThreadMinimum(m_ThreadPool, InThreadCount); + SetThreadpoolThreadMaximum(m_ThreadPool, InThreadCount * 2); + + InitializeThreadpoolEnvironment(&m_CallbackEnvironment); + + m_CleanupGroup = CreateThreadpoolCleanupGroup(); + + SetThreadpoolCallbackPool(&m_CallbackEnvironment, m_ThreadPool); + + SetThreadpoolCallbackCleanupGroup(&m_CallbackEnvironment, m_CleanupGroup, NULL); +} + +WinIoThreadPool::~WinIoThreadPool() +{ + CloseThreadpool(m_ThreadPool); +} + +void +WinIoThreadPool::CreateIocp(HANDLE IoHandle, PTP_WIN32_IO_CALLBACK Callback, void* Context) +{ + m_ThreadPoolIo = CreateThreadpoolIo(IoHandle, Callback, Context, &m_CallbackEnvironment); +} + +} // namespace zen diff --git a/zencore/iothreadpool.h b/zencore/iothreadpool.h new file mode 100644 index 000000000..f64868540 --- /dev/null +++ b/zencore/iothreadpool.h @@ -0,0 +1,31 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/windows.h> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +// +// Thread pool. Implemented in terms of Windows thread pool right now, will +// need a cross-platform implementation eventually +// + +class WinIoThreadPool +{ +public: + WinIoThreadPool(int InThreadCount); + ~WinIoThreadPool(); + + void CreateIocp(HANDLE IoHandle, PTP_WIN32_IO_CALLBACK Callback, void* Context); + inline PTP_IO Iocp() const { return m_ThreadPoolIo; } + +private: + PTP_POOL m_ThreadPool = nullptr; + PTP_CLEANUP_GROUP m_CleanupGroup = nullptr; + PTP_IO m_ThreadPoolIo = nullptr; + TP_CALLBACK_ENVIRON m_CallbackEnvironment; +}; + +} // namespace zen diff --git a/zencore/md5.cpp b/zencore/md5.cpp new file mode 100644 index 000000000..228c0feff --- /dev/null +++ b/zencore/md5.cpp @@ -0,0 +1,446 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/md5.h> +#include <zencore/string.h> +#include <zencore/zencore.h> + +#include <doctest/doctest.h> +#include <string.h> + +// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN +#ifndef _MSC_VER +# include <endian.h> +#endif + +/* + ********************************************************************** + ** md5.h -- Header file for implementation of MD5 ** + ** RSA Data Security, Inc. MD5 Message Digest Algorithm ** + ** Created: 2/17/90 RLR ** + ** Revised: 12/27/90 SRD,AJ,BSK,JT Reference C version ** + ** Revised (for MD5): RLR 4/27/91 ** + ** -- G modified to have y&~z instead of y&z ** + ** -- FF, GG, HH modified to add in last register done ** + ** -- Access pattern: round 2 works mod 5, round 3 works mod 3 ** + ** -- distinct additive constant for each step ** + ** -- round 4 added, working mod 7 ** + ********************************************************************** + */ + +/* + ********************************************************************** + ** Copyright (C) 1990, RSA Data Security, Inc. All rights reserved. ** + ** ** + ** License to copy and use this software is granted provided that ** + ** it is identified as the "RSA Data Security, Inc. MD5 Message ** + ** Digest Algorithm" in all material mentioning or referencing this ** + ** software or this function. ** + ** ** + ** License is also granted to make and use derivative works ** + ** provided that such works are identified as "derived from the RSA ** + ** Data Security, Inc. MD5 Message Digest Algorithm" in all ** + ** material mentioning or referencing the derived work. ** + ** ** + ** RSA Data Security, Inc. makes no representations concerning ** + ** either the merchantability of this software or the suitability ** + ** of this software for any particular purpose. It is provided "as ** + ** is" without express or implied warranty of any kind. ** + ** ** + ** These notices must be retained in any copies of any part of this ** + ** documentation and/or software. ** + ********************************************************************** + */ + +/* Data structure for MD5 (Message Digest) computation */ +struct MD5_CTX +{ + uint32_t i[2]; /* number of _bits_ handled mod 2^64 */ + uint32_t buf[4]; /* scratch buffer */ + unsigned char in[64]; /* input buffer */ + unsigned char digest[16]; /* actual digest after MD5Final call */ +}; + +void MD5Init(); +void MD5Update(); +void MD5Final(); + +/* + ********************************************************************** + ** End of md5.h ** + ******************************* (cut) ******************************** + */ + +/* + ********************************************************************** + ** md5.c ** + ** RSA Data Security, Inc. MD5 Message Digest Algorithm ** + ** Created: 2/17/90 RLR ** + ** Revised: 1/91 SRD,AJ,BSK,JT Reference C Version ** + ********************************************************************** + */ + +/* + ********************************************************************** + ** Copyright (C) 1990, RSA Data Security, Inc. All rights reserved. ** + ** ** + ** License to copy and use this software is granted provided that ** + ** it is identified as the "RSA Data Security, Inc. MD5 Message ** + ** Digest Algorithm" in all material mentioning or referencing this ** + ** software or this function. ** + ** ** + ** License is also granted to make and use derivative works ** + ** provided that such works are identified as "derived from the RSA ** + ** Data Security, Inc. MD5 Message Digest Algorithm" in all ** + ** material mentioning or referencing the derived work. ** + ** ** + ** RSA Data Security, Inc. makes no representations concerning ** + ** either the merchantability of this software or the suitability ** + ** of this software for any particular purpose. It is provided "as ** + ** is" without express or implied warranty of any kind. ** + ** ** + ** These notices must be retained in any copies of any part of this ** + ** documentation and/or software. ** + ********************************************************************** + */ + +/* -- include the following line if the md5.h header file is separate -- */ +/* #include "md5.h" */ + +/* forward declaration */ +static void Transform(uint32_t* buf, uint32_t* in); + +static unsigned char PADDING[64] = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +/* F, G and H are basic MD5 functions: selection, majority, parity */ +#define F(x, y, z) (((x) & (y)) | ((~x) & (z))) +#define G(x, y, z) (((x) & (z)) | ((y) & (~z))) +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define I(x, y, z) ((y) ^ ((x) | (~z))) + +/* ROTATE_LEFT rotates x left n bits */ +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) + +/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4 */ +/* Rotation is separate from addition to prevent recomputation */ +#define FF(a, b, c, d, x, s, ac) \ + { \ + (a) += F((b), (c), (d)) + (x) + (uint32_t)(ac); \ + (a) = ROTATE_LEFT((a), (s)); \ + (a) += (b); \ + } +#define GG(a, b, c, d, x, s, ac) \ + { \ + (a) += G((b), (c), (d)) + (x) + (uint32_t)(ac); \ + (a) = ROTATE_LEFT((a), (s)); \ + (a) += (b); \ + } +#define HH(a, b, c, d, x, s, ac) \ + { \ + (a) += H((b), (c), (d)) + (x) + (uint32_t)(ac); \ + (a) = ROTATE_LEFT((a), (s)); \ + (a) += (b); \ + } +#define II(a, b, c, d, x, s, ac) \ + { \ + (a) += I((b), (c), (d)) + (x) + (uint32_t)(ac); \ + (a) = ROTATE_LEFT((a), (s)); \ + (a) += (b); \ + } + +void +MD5Init(MD5_CTX* mdContext) +{ + mdContext->i[0] = mdContext->i[1] = (uint32_t)0; + + /* Load magic initialization constants. + */ + mdContext->buf[0] = (uint32_t)0x67452301; + mdContext->buf[1] = (uint32_t)0xefcdab89; + mdContext->buf[2] = (uint32_t)0x98badcfe; + mdContext->buf[3] = (uint32_t)0x10325476; +} + +void +MD5Update(MD5_CTX* mdContext, unsigned char* inBuf, unsigned int inLen) +{ + uint32_t in[16]; + int mdi; + unsigned int i, ii; + + /* compute number of bytes mod 64 */ + mdi = (int)((mdContext->i[0] >> 3) & 0x3F); + + /* update number of bits */ + if ((mdContext->i[0] + ((uint32_t)inLen << 3)) < mdContext->i[0]) + mdContext->i[1]++; + mdContext->i[0] += ((uint32_t)inLen << 3); + mdContext->i[1] += ((uint32_t)inLen >> 29); + + while (inLen--) + { + /* add new character to buffer, increment mdi */ + mdContext->in[mdi++] = *inBuf++; + + /* transform if necessary */ + if (mdi == 0x40) + { + for (i = 0, ii = 0; i < 16; i++, ii += 4) + in[i] = (((uint32_t)mdContext->in[ii + 3]) << 24) | (((uint32_t)mdContext->in[ii + 2]) << 16) | + (((uint32_t)mdContext->in[ii + 1]) << 8) | ((uint32_t)mdContext->in[ii]); + Transform(mdContext->buf, in); + mdi = 0; + } + } +} + +void +MD5Final(MD5_CTX* mdContext) +{ + uint32_t in[16]; + int mdi; + unsigned int i, ii; + unsigned int padLen; + + /* save number of bits */ + in[14] = mdContext->i[0]; + in[15] = mdContext->i[1]; + + /* compute number of bytes mod 64 */ + mdi = (int)((mdContext->i[0] >> 3) & 0x3F); + + /* pad out to 56 mod 64 */ + padLen = (mdi < 56) ? (56 - mdi) : (120 - mdi); + MD5Update(mdContext, PADDING, padLen); + + /* append length in bits and transform */ + for (i = 0, ii = 0; i < 14; i++, ii += 4) + in[i] = (((uint32_t)mdContext->in[ii + 3]) << 24) | (((uint32_t)mdContext->in[ii + 2]) << 16) | + (((uint32_t)mdContext->in[ii + 1]) << 8) | ((uint32_t)mdContext->in[ii]); + Transform(mdContext->buf, in); + + /* store buffer in digest */ + for (i = 0, ii = 0; i < 4; i++, ii += 4) + { + mdContext->digest[ii] = (unsigned char)(mdContext->buf[i] & 0xFF); + mdContext->digest[ii + 1] = (unsigned char)((mdContext->buf[i] >> 8) & 0xFF); + mdContext->digest[ii + 2] = (unsigned char)((mdContext->buf[i] >> 16) & 0xFF); + mdContext->digest[ii + 3] = (unsigned char)((mdContext->buf[i] >> 24) & 0xFF); + } +} + +/* Basic MD5 step. Transform buf based on in. + */ +static void +Transform(uint32_t* buf, uint32_t* in) +{ + uint32_t a = buf[0], b = buf[1], c = buf[2], d = buf[3]; + + /* Round 1 */ +#define S11 7 +#define S12 12 +#define S13 17 +#define S14 22 + FF(a, b, c, d, in[0], S11, 3614090360); /* 1 */ + FF(d, a, b, c, in[1], S12, 3905402710); /* 2 */ + FF(c, d, a, b, in[2], S13, 606105819); /* 3 */ + FF(b, c, d, a, in[3], S14, 3250441966); /* 4 */ + FF(a, b, c, d, in[4], S11, 4118548399); /* 5 */ + FF(d, a, b, c, in[5], S12, 1200080426); /* 6 */ + FF(c, d, a, b, in[6], S13, 2821735955); /* 7 */ + FF(b, c, d, a, in[7], S14, 4249261313); /* 8 */ + FF(a, b, c, d, in[8], S11, 1770035416); /* 9 */ + FF(d, a, b, c, in[9], S12, 2336552879); /* 10 */ + FF(c, d, a, b, in[10], S13, 4294925233); /* 11 */ + FF(b, c, d, a, in[11], S14, 2304563134); /* 12 */ + FF(a, b, c, d, in[12], S11, 1804603682); /* 13 */ + FF(d, a, b, c, in[13], S12, 4254626195); /* 14 */ + FF(c, d, a, b, in[14], S13, 2792965006); /* 15 */ + FF(b, c, d, a, in[15], S14, 1236535329); /* 16 */ + + /* Round 2 */ +#define S21 5 +#define S22 9 +#define S23 14 +#define S24 20 + GG(a, b, c, d, in[1], S21, 4129170786); /* 17 */ + GG(d, a, b, c, in[6], S22, 3225465664); /* 18 */ + GG(c, d, a, b, in[11], S23, 643717713); /* 19 */ + GG(b, c, d, a, in[0], S24, 3921069994); /* 20 */ + GG(a, b, c, d, in[5], S21, 3593408605); /* 21 */ + GG(d, a, b, c, in[10], S22, 38016083); /* 22 */ + GG(c, d, a, b, in[15], S23, 3634488961); /* 23 */ + GG(b, c, d, a, in[4], S24, 3889429448); /* 24 */ + GG(a, b, c, d, in[9], S21, 568446438); /* 25 */ + GG(d, a, b, c, in[14], S22, 3275163606); /* 26 */ + GG(c, d, a, b, in[3], S23, 4107603335); /* 27 */ + GG(b, c, d, a, in[8], S24, 1163531501); /* 28 */ + GG(a, b, c, d, in[13], S21, 2850285829); /* 29 */ + GG(d, a, b, c, in[2], S22, 4243563512); /* 30 */ + GG(c, d, a, b, in[7], S23, 1735328473); /* 31 */ + GG(b, c, d, a, in[12], S24, 2368359562); /* 32 */ + + /* Round 3 */ +#define S31 4 +#define S32 11 +#define S33 16 +#define S34 23 + HH(a, b, c, d, in[5], S31, 4294588738); /* 33 */ + HH(d, a, b, c, in[8], S32, 2272392833); /* 34 */ + HH(c, d, a, b, in[11], S33, 1839030562); /* 35 */ + HH(b, c, d, a, in[14], S34, 4259657740); /* 36 */ + HH(a, b, c, d, in[1], S31, 2763975236); /* 37 */ + HH(d, a, b, c, in[4], S32, 1272893353); /* 38 */ + HH(c, d, a, b, in[7], S33, 4139469664); /* 39 */ + HH(b, c, d, a, in[10], S34, 3200236656); /* 40 */ + HH(a, b, c, d, in[13], S31, 681279174); /* 41 */ + HH(d, a, b, c, in[0], S32, 3936430074); /* 42 */ + HH(c, d, a, b, in[3], S33, 3572445317); /* 43 */ + HH(b, c, d, a, in[6], S34, 76029189); /* 44 */ + HH(a, b, c, d, in[9], S31, 3654602809); /* 45 */ + HH(d, a, b, c, in[12], S32, 3873151461); /* 46 */ + HH(c, d, a, b, in[15], S33, 530742520); /* 47 */ + HH(b, c, d, a, in[2], S34, 3299628645); /* 48 */ + + /* Round 4 */ +#define S41 6 +#define S42 10 +#define S43 15 +#define S44 21 + II(a, b, c, d, in[0], S41, 4096336452); /* 49 */ + II(d, a, b, c, in[7], S42, 1126891415); /* 50 */ + II(c, d, a, b, in[14], S43, 2878612391); /* 51 */ + II(b, c, d, a, in[5], S44, 4237533241); /* 52 */ + II(a, b, c, d, in[12], S41, 1700485571); /* 53 */ + II(d, a, b, c, in[3], S42, 2399980690); /* 54 */ + II(c, d, a, b, in[10], S43, 4293915773); /* 55 */ + II(b, c, d, a, in[1], S44, 2240044497); /* 56 */ + II(a, b, c, d, in[8], S41, 1873313359); /* 57 */ + II(d, a, b, c, in[15], S42, 4264355552); /* 58 */ + II(c, d, a, b, in[6], S43, 2734768916); /* 59 */ + II(b, c, d, a, in[13], S44, 1309151649); /* 60 */ + II(a, b, c, d, in[4], S41, 4149444226); /* 61 */ + II(d, a, b, c, in[11], S42, 3174756917); /* 62 */ + II(c, d, a, b, in[2], S43, 718787259); /* 63 */ + II(b, c, d, a, in[9], S44, 3951481745); /* 64 */ + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +/* + ********************************************************************** + ** End of md5.c ** + ******************************* (cut) ******************************** + */ + +#undef FF +#undef GG +#undef HH +#undef II +#undef F +#undef G +#undef H +#undef I + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +MD5 MD5::Zero; // Initialized to all zeroes + +////////////////////////////////////////////////////////////////////////// + +MD5Stream::MD5Stream() +{ + Reset(); +} + +void +MD5Stream::Reset() +{ +} + +MD5Stream& +MD5Stream::Append(const void* Data, size_t ByteCount) +{ + ZEN_UNUSED(Data); + ZEN_UNUSED(ByteCount); + + return *this; +} + +MD5 +MD5Stream::GetHash() +{ + MD5 md5{}; + + return md5; +} + +////////////////////////////////////////////////////////////////////////// + +MD5 +MD5::HashMemory(const void* data, size_t byteCount) +{ + return MD5Stream().Append(data, byteCount).GetHash(); +} + +MD5 +MD5::FromHexString(const char* string) +{ + MD5 md5; + + ParseHexBytes(string, 40, md5.Hash); + + return md5; +} + +const char* +MD5::ToHexString(char* outString /* 40 characters + NUL terminator */) const +{ + ToHexBytes(Hash, sizeof(MD5), outString); + outString[2 * sizeof(MD5)] = '\0'; + + return outString; +} + +StringBuilderBase& +MD5::ToHexString(StringBuilderBase& outBuilder) const +{ + char str[41]; + ToHexString(str); + + outBuilder.AppendRange(str, &str[40]); + + return outBuilder; +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +void +md5_forcelink() +{ +} + +doctest::String +toString(const MD5& value) +{ + char md5text[2 * sizeof(MD5) + 1]; + value.ToHexString(md5text); + + return md5text; +} + +TEST_CASE("MD5") +{ +} + +} // namespace zen diff --git a/zencore/memory.cpp b/zencore/memory.cpp new file mode 100644 index 000000000..63d61f5e1 --- /dev/null +++ b/zencore/memory.cpp @@ -0,0 +1,165 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <malloc.h> +#include <zencore/intmath.h> +#include <zencore/memory.h> + +#include <doctest/doctest.h> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +MemoryArena::MemoryArena() +{ +} + +MemoryArena::~MemoryArena() +{ +} + +void* +MemoryArena::Alloc(size_t size, size_t alignment) +{ + return _mm_malloc(size, alignment); +} + +void +MemoryArena::Free(void* ptr) +{ + if (ptr) + _mm_free(ptr); +} + +////////////////////////////////////////////////////////////////////////// + +void* +Memory::Alloc(size_t size, size_t alignment) +{ + return _mm_malloc(size, alignment); +} + +void +Memory::Free(void* ptr) +{ + if (ptr) + _mm_free(ptr); +} + +////////////////////////////////////////////////////////////////////////// + +ChunkingLinearAllocator::ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment) +: m_ChunkSize(ChunkSize) +, m_ChunkAlignment(ChunkAlignment) +{ +} + +ChunkingLinearAllocator::~ChunkingLinearAllocator() +{ + Reset(); +} + +void +ChunkingLinearAllocator::Reset() +{ + for (void* ChunkEntry : m_ChunkList) + { + Memory::Free(ChunkEntry); + } + m_ChunkList.clear(); + + m_ChunkCursor = nullptr; + m_ChunkBytesRemain = 0; +} + +void* +ChunkingLinearAllocator::Alloc(size_t Size, size_t Alignment) +{ + ZEN_ASSERT_SLOW(zen::IsPow2(Alignment)); + + // This could be improved in a bunch of ways + // + // * We pessimistically allocate memory even though there may be enough memory available for a single allocation due to the way we take + // alignment into account below + // * The block allocation size could be chosen to minimize slack for the case when multiple oversize allocations are made rather than + // minimizing the number of chunks + // * ... + + const uint64_t AllocationSize = zen::RoundUp(Size, Alignment); + + if (m_ChunkBytesRemain < (AllocationSize + Alignment - 1)) + { + const uint64_t ChunkSize = zen::RoundUp(zen::Max(m_ChunkSize, Size), m_ChunkSize); + void* ChunkPtr = Memory::Alloc(ChunkSize, m_ChunkAlignment); + m_ChunkCursor = reinterpret_cast<uint8_t*>(ChunkPtr); + m_ChunkBytesRemain = ChunkSize; + m_ChunkList.push_back(ChunkPtr); + } + + const uint64_t AlignFixup = (Alignment - reinterpret_cast<uintptr_t>(m_ChunkCursor)) & (Alignment - 1); + void* ReturnPtr = m_ChunkCursor + AlignFixup; + const uint64_t Delta = AlignFixup + AllocationSize; + + ZEN_ASSERT_SLOW(m_ChunkBytesRemain >= Delta); + + m_ChunkCursor += Delta; + m_ChunkBytesRemain -= Delta; + + ZEN_ASSERT_SLOW(IsPointerAligned(ReturnPtr, Alignment)); + + return ReturnPtr; +} + +////////////////////////////////////////////////////////////////////////// +// +// Unit tests +// + +TEST_CASE("ChunkingLinearAllocator") +{ + ChunkingLinearAllocator Allocator(4096); + + void* p1 = Allocator.Alloc(1, 1); + void* p2 = Allocator.Alloc(1, 1); + + CHECK(p1 != p2); + + void* p3 = Allocator.Alloc(1, 4); + CHECK(IsPointerAligned(p3, 4)); + + void* p3_2 = Allocator.Alloc(1, 4); + CHECK(IsPointerAligned(p3_2, 4)); + + void* p4 = Allocator.Alloc(1, 8); + CHECK(IsPointerAligned(p4, 8)); + + for (int i = 0; i < 100; ++i) + { + void* p0 = Allocator.Alloc(64); + ZEN_UNUSED(p0); + } +} + +TEST_CASE("MemoryView") +{ + { + uint8_t Array1[16]; + MemoryView View1 = MakeMemoryView(Array1); + CHECK(View1.GetSize() == 16); + } + + { + uint32_t Array2[16]; + MemoryView View2 = MakeMemoryView(Array2); + CHECK(View2.GetSize() == 64); + } + + CHECK(MakeMemoryView<float>({1.0f, 1.2f}).GetSize() == 8); +} + +void +memory_forcelink() +{ +} + +} // namespace zen diff --git a/zencore/refcount.cpp b/zencore/refcount.cpp new file mode 100644 index 000000000..943635552 --- /dev/null +++ b/zencore/refcount.cpp @@ -0,0 +1,96 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/refcount.h> + +#include <doctest/doctest.h> +#include <functional> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +struct TestRefClass : public RefCounted +{ + ~TestRefClass() + { + if (OnDestroy) + OnDestroy(); + } + + using RefCounted::RefCount; + + std::function<void()> OnDestroy; +}; + +void +refcount_forcelink() +{ +} + +TEST_CASE("RefPtr") +{ + RefPtr<TestRefClass> Ref; + Ref = new TestRefClass; + + bool IsDestroyed = false; + Ref->OnDestroy = [&] { IsDestroyed = true; }; + + CHECK(IsDestroyed == false); + CHECK(Ref->RefCount() == 1); + + RefPtr<TestRefClass> Ref2; + Ref2 = Ref; + + CHECK(IsDestroyed == false); + CHECK(Ref->RefCount() == 2); + + RefPtr<TestRefClass> Ref3; + Ref2 = Ref3; + + CHECK(IsDestroyed == false); + CHECK(Ref->RefCount() == 1); + Ref = Ref3; + + CHECK(IsDestroyed == true); +} + +TEST_CASE("RefPtr on Stack allocated object") +{ + bool IsDestroyed = false; + + { + TestRefClass StackRefClass; + + StackRefClass.OnDestroy = [&] { IsDestroyed = true; }; + + CHECK(StackRefClass.RefCount() == 1); // Stack allocated objects should have +1 ref + + RefPtr<TestRefClass> Ref{&StackRefClass}; + + CHECK(IsDestroyed == false); + CHECK(StackRefClass.RefCount() == 2); + + RefPtr<TestRefClass> Ref2; + Ref2 = Ref; + + CHECK(IsDestroyed == false); + CHECK(StackRefClass.RefCount() == 3); + + RefPtr<TestRefClass> Ref3; + Ref2 = Ref3; + + CHECK(IsDestroyed == false); + CHECK(StackRefClass.RefCount() == 2); + + Ref = Ref3; + CHECK(IsDestroyed == false); + CHECK(StackRefClass.RefCount() == 1); + } + + CHECK(IsDestroyed == true); +} + +} // namespace zen diff --git a/zencore/sha1.cpp b/zencore/sha1.cpp new file mode 100644 index 000000000..3cc2f5cdf --- /dev/null +++ b/zencore/sha1.cpp @@ -0,0 +1,439 @@ +// ////////////////////////////////////////////////////////// +// sha1.cpp +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#include <zencore/sha1.h> +#include <zencore/string.h> +#include <zencore/zencore.h> + +#include <doctest/doctest.h> +#include <string.h> + +// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN +#ifndef _MSC_VER +# include <endian.h> +#endif + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +SHA1 SHA1::Zero; // Initialized to all zeroes + +////////////////////////////////////////////////////////////////////////// + +SHA1Stream::SHA1Stream() +{ + Reset(); +} + +void +SHA1Stream::Reset() +{ + m_NumBytes = 0; + m_BufferSize = 0; + + // according to RFC 1321 + m_Hash[0] = 0x67452301; + m_Hash[1] = 0xefcdab89; + m_Hash[2] = 0x98badcfe; + m_Hash[3] = 0x10325476; + m_Hash[4] = 0xc3d2e1f0; +} + +namespace { + // mix functions for processBlock() + inline uint32_t f1(uint32_t b, uint32_t c, uint32_t d) + { + return d ^ (b & (c ^ d)); // original: f = (b & c) | ((~b) & d); + } + + inline uint32_t f2(uint32_t b, uint32_t c, uint32_t d) { return b ^ c ^ d; } + + inline uint32_t f3(uint32_t b, uint32_t c, uint32_t d) { return (b & c) | (b & d) | (c & d); } + + inline uint32_t rotate(uint32_t a, uint32_t c) { return (a << c) | (a >> (32 - c)); } + + inline uint32_t swap(uint32_t x) + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#endif +#ifdef MSC_VER + return _byteswap_ulong(x); +#endif + + return (x >> 24) | ((x >> 8) & 0x0000FF00) | ((x << 8) & 0x00FF0000) | (x << 24); + } +} // namespace + +/// process 64 bytes +void +SHA1Stream::ProcessBlock(const void* data) +{ + // get last hash + uint32_t a = m_Hash[0]; + uint32_t b = m_Hash[1]; + uint32_t c = m_Hash[2]; + uint32_t d = m_Hash[3]; + uint32_t e = m_Hash[4]; + + // data represented as 16x 32-bit words + const uint32_t* input = (uint32_t*)data; + // convert to big endian + uint32_t words[80]; + for (int i = 0; i < 16; i++) +#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) + words[i] = input[i]; +#else + words[i] = swap(input[i]); +#endif + + // extend to 80 words + for (int i = 16; i < 80; i++) + words[i] = rotate(words[i - 3] ^ words[i - 8] ^ words[i - 14] ^ words[i - 16], 1); + + // first round + for (int i = 0; i < 4; i++) + { + int offset = 5 * i; + e += rotate(a, 5) + f1(b, c, d) + words[offset] + 0x5a827999; + b = rotate(b, 30); + d += rotate(e, 5) + f1(a, b, c) + words[offset + 1] + 0x5a827999; + a = rotate(a, 30); + c += rotate(d, 5) + f1(e, a, b) + words[offset + 2] + 0x5a827999; + e = rotate(e, 30); + b += rotate(c, 5) + f1(d, e, a) + words[offset + 3] + 0x5a827999; + d = rotate(d, 30); + a += rotate(b, 5) + f1(c, d, e) + words[offset + 4] + 0x5a827999; + c = rotate(c, 30); + } + + // second round + for (int i = 4; i < 8; i++) + { + int offset = 5 * i; + e += rotate(a, 5) + f2(b, c, d) + words[offset] + 0x6ed9eba1; + b = rotate(b, 30); + d += rotate(e, 5) + f2(a, b, c) + words[offset + 1] + 0x6ed9eba1; + a = rotate(a, 30); + c += rotate(d, 5) + f2(e, a, b) + words[offset + 2] + 0x6ed9eba1; + e = rotate(e, 30); + b += rotate(c, 5) + f2(d, e, a) + words[offset + 3] + 0x6ed9eba1; + d = rotate(d, 30); + a += rotate(b, 5) + f2(c, d, e) + words[offset + 4] + 0x6ed9eba1; + c = rotate(c, 30); + } + + // third round + for (int i = 8; i < 12; i++) + { + int offset = 5 * i; + e += rotate(a, 5) + f3(b, c, d) + words[offset] + 0x8f1bbcdc; + b = rotate(b, 30); + d += rotate(e, 5) + f3(a, b, c) + words[offset + 1] + 0x8f1bbcdc; + a = rotate(a, 30); + c += rotate(d, 5) + f3(e, a, b) + words[offset + 2] + 0x8f1bbcdc; + e = rotate(e, 30); + b += rotate(c, 5) + f3(d, e, a) + words[offset + 3] + 0x8f1bbcdc; + d = rotate(d, 30); + a += rotate(b, 5) + f3(c, d, e) + words[offset + 4] + 0x8f1bbcdc; + c = rotate(c, 30); + } + + // fourth round + for (int i = 12; i < 16; i++) + { + int offset = 5 * i; + e += rotate(a, 5) + f2(b, c, d) + words[offset] + 0xca62c1d6; + b = rotate(b, 30); + d += rotate(e, 5) + f2(a, b, c) + words[offset + 1] + 0xca62c1d6; + a = rotate(a, 30); + c += rotate(d, 5) + f2(e, a, b) + words[offset + 2] + 0xca62c1d6; + e = rotate(e, 30); + b += rotate(c, 5) + f2(d, e, a) + words[offset + 3] + 0xca62c1d6; + d = rotate(d, 30); + a += rotate(b, 5) + f2(c, d, e) + words[offset + 4] + 0xca62c1d6; + c = rotate(c, 30); + } + + // update hash + m_Hash[0] += a; + m_Hash[1] += b; + m_Hash[2] += c; + m_Hash[3] += d; + m_Hash[4] += e; +} + +/// add arbitrary number of bytes +SHA1Stream& +SHA1Stream::Append(const void* data, size_t byteCount) +{ + const uint8_t* current = (const uint8_t*)data; + + if (m_BufferSize > 0) + { + while (byteCount > 0 && m_BufferSize < BlockSize) + { + m_Buffer[m_BufferSize++] = *current++; + byteCount--; + } + } + + // full buffer + if (m_BufferSize == BlockSize) + { + ProcessBlock((void*)m_Buffer); + m_NumBytes += BlockSize; + m_BufferSize = 0; + } + + // no more data ? + if (byteCount == 0) + return *this; + + // process full blocks + while (byteCount >= BlockSize) + { + ProcessBlock(current); + current += BlockSize; + m_NumBytes += BlockSize; + byteCount -= BlockSize; + } + + // keep remaining bytes in buffer + while (byteCount > 0) + { + m_Buffer[m_BufferSize++] = *current++; + byteCount--; + } + + return *this; +} + +/// process final block, less than 64 bytes +void +SHA1Stream::ProcessBuffer() +{ + // the input bytes are considered as bits strings, where the first bit is the most significant bit of the byte + + // - append "1" bit to message + // - append "0" bits until message length in bit mod 512 is 448 + // - append length as 64 bit integer + + // number of bits + size_t paddedLength = m_BufferSize * 8; + + // plus one bit set to 1 (always appended) + paddedLength++; + + // number of bits must be (numBits % 512) = 448 + size_t lower11Bits = paddedLength & 511; + if (lower11Bits <= 448) + paddedLength += 448 - lower11Bits; + else + paddedLength += 512 + 448 - lower11Bits; + // convert from bits to bytes + paddedLength /= 8; + + // only needed if additional data flows over into a second block + unsigned char extra[BlockSize]; + + // append a "1" bit, 128 => binary 10000000 + if (m_BufferSize < BlockSize) + m_Buffer[m_BufferSize] = 128; + else + extra[0] = 128; + + size_t i; + for (i = m_BufferSize + 1; i < BlockSize; i++) + m_Buffer[i] = 0; + for (; i < paddedLength; i++) + extra[i - BlockSize] = 0; + + // add message length in bits as 64 bit number + uint64_t msgBits = 8 * (m_NumBytes + m_BufferSize); + // find right position + unsigned char* addLength; + if (paddedLength < BlockSize) + addLength = m_Buffer + paddedLength; + else + addLength = extra + paddedLength - BlockSize; + + // must be big endian + *addLength++ = (unsigned char)((msgBits >> 56) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 48) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 40) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 32) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 24) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 16) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 8) & 0xFF); + *addLength = (unsigned char)(msgBits & 0xFF); + + // process blocks + ProcessBlock(m_Buffer); + // flowed over into a second block ? + if (paddedLength > BlockSize) + ProcessBlock(extra); +} + +/// return latest hash as bytes +SHA1 +SHA1Stream::GetHash() +{ + SHA1 sha1; + // save old hash if buffer is partially filled + uint32_t oldHash[HashValues]; + for (int i = 0; i < HashValues; i++) + oldHash[i] = m_Hash[i]; + + // process remaining bytes + ProcessBuffer(); + + unsigned char* current = sha1.Hash; + for (int i = 0; i < HashValues; i++) + { + *current++ = (m_Hash[i] >> 24) & 0xFF; + *current++ = (m_Hash[i] >> 16) & 0xFF; + *current++ = (m_Hash[i] >> 8) & 0xFF; + *current++ = m_Hash[i] & 0xFF; + + // restore old hash + m_Hash[i] = oldHash[i]; + } + + return sha1; +} + +/// compute SHA1 of a memory block +SHA1 +SHA1Stream::Compute(const void* data, size_t byteCount) +{ + Reset(); + Append(data, byteCount); + return GetHash(); +} + +SHA1 +SHA1::HashMemory(const void* data, size_t byteCount) +{ + return SHA1Stream().Append(data, byteCount).GetHash(); +} + +SHA1 +SHA1::FromHexString(const char* string) +{ + SHA1 sha1; + + ParseHexBytes(string, 40, sha1.Hash); + + return sha1; +} + +const char* +SHA1::ToHexString(char* outString /* 40 characters + NUL terminator */) const +{ + ToHexBytes(Hash, sizeof(SHA1), outString); + outString[2 * sizeof(SHA1)] = '\0'; + + return outString; +} + +StringBuilderBase& +SHA1::ToHexString(StringBuilderBase& outBuilder) const +{ + char str[41]; + ToHexString(str); + + outBuilder.AppendRange(str, &str[40]); + + return outBuilder; +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +void +sha1_forcelink() +{ +} + +doctest::String +toString(const SHA1& value) +{ + char sha1text[2 * sizeof(SHA1) + 1]; + value.ToHexString(sha1text); + + return sha1text; +} + +TEST_CASE("SHA1") +{ + uint8_t sha1_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, + 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09}; + SHA1 sha1z; + memcpy(sha1z.Hash, sha1_empty, sizeof sha1z.Hash); + + SUBCASE("Empty string") + { + SHA1 sha1 = SHA1::HashMemory(nullptr, 0); + + CHECK(sha1 == sha1z); + } + + SUBCASE("Empty stream") + { + SHA1Stream sha1s; + sha1s.Append(nullptr, 0); + sha1s.Append(nullptr, 0); + sha1s.Append(nullptr, 0); + CHECK(sha1s.GetHash() == sha1z); + } + + SUBCASE("SHA1 from string") + { + const SHA1 sha1empty = SHA1::FromHexString("da39a3ee5e6b4b0d3255bfef95601890afd80709"); + + CHECK(sha1z == sha1empty); + } + + SUBCASE("SHA1 to string") + { + char sha1str[41]; + sha1z.ToHexString(sha1str); + + CHECK(StringEquals(sha1str, "da39a3ee5e6b4b0d3255bfef95601890afd80709")); + } + + SUBCASE("Hash ABC") + { + const SHA1 sha1abc = SHA1::FromHexString("3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"); + + SHA1Stream sha1s; + + sha1s.Append("A", 1); + sha1s.Append("B", 1); + sha1s.Append("C", 1); + CHECK(sha1s.GetHash() == sha1abc); + + sha1s.Reset(); + sha1s.Append("AB", 2); + sha1s.Append("C", 1); + CHECK(sha1s.GetHash() == sha1abc); + + sha1s.Reset(); + sha1s.Append("ABC", 3); + CHECK(sha1s.GetHash() == sha1abc); + + sha1s.Reset(); + sha1s.Append("A", 1); + sha1s.Append("BC", 2); + CHECK(sha1s.GetHash() == sha1abc); + } +} + +} // namespace zen diff --git a/zencore/sharedbuffer.cpp b/zencore/sharedbuffer.cpp new file mode 100644 index 000000000..bc991053d --- /dev/null +++ b/zencore/sharedbuffer.cpp @@ -0,0 +1,110 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/sharedbuffer.h> + +#include <doctest/doctest.h> +#include <memory.h> + +#include <atlfile.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +BufferOwner::~BufferOwner() +{ + if (m_IsOwned) + { + Memory::Free(m_Data); + } +} + +////////////////////////////////////////////////////////////////////////// + +UniqueBuffer +UniqueBuffer::Alloc(uint64_t Size) +{ + void* Buffer = Memory::Alloc(Size, 16); + BufferOwner* Owner = new BufferOwner(Buffer, Size, /* owned */ true); + + return UniqueBuffer(Owner); +} + +UniqueBuffer +UniqueBuffer::MakeView(void* DataPtr, uint64_t Size) +{ + return UniqueBuffer(new BufferOwner(DataPtr, Size, /* owned */ false)); +} + +UniqueBuffer::UniqueBuffer(BufferOwner* Owner) : m_buffer(Owner) +{ +} + +////////////////////////////////////////////////////////////////////////// + +SharedBuffer::SharedBuffer(UniqueBuffer&& InBuffer) : m_buffer(std::move(InBuffer.m_buffer)) +{ +} + +void +SharedBuffer::MakeOwned() +{ + if (IsOwned() || !m_buffer) + return; + + const uint64_t Size = m_buffer->m_Size; + void* Buffer = Memory::Alloc(Size, 16); + auto NewOwner = new BufferOwner(Buffer, Size, /* owned */ true); + + memcpy(Buffer, m_buffer->m_Data, Size); + + m_buffer = NewOwner; +} + +SharedBuffer +SharedBuffer::MakeView(MemoryView View, SharedBuffer Buffer) +{ + // Todo: verify that view is within the shared buffer + + return SharedBuffer(new BufferOwner(const_cast<void*>(View.GetData()), View.GetSize(), /* owned */ false, Buffer.m_buffer)); +} + +SharedBuffer +SharedBuffer::MakeView(const void* Data, uint64_t Size) +{ + return SharedBuffer(new BufferOwner(const_cast<void*>(Data), Size, /* owned */ false)); +} + +SharedBuffer +SharedBuffer::Clone() +{ + const uint64_t Size = GetSize(); + void* Buffer = Memory::Alloc(Size, 16); + auto NewOwner = new BufferOwner(Buffer, Size, /* owned */ true); + memcpy(Buffer, m_buffer->m_Data, Size); + + return SharedBuffer(NewOwner); +} + +SharedBuffer +SharedBuffer::Clone(MemoryView View) +{ + const uint64_t Size = View.GetSize(); + void* Buffer = Memory::Alloc(Size, 16); + auto NewOwner = new BufferOwner(Buffer, Size, /* owned */ true); + memcpy(Buffer, View.GetData(), Size); + + return SharedBuffer(NewOwner); +} + +////////////////////////////////////////////////////////////////////////// + +void +sharedbuffer_forcelink() +{ +} + +TEST_CASE("SharedBuffer") +{ +} + +} // namespace zen diff --git a/zencore/snapshot_manifest.cpp b/zencore/snapshot_manifest.cpp new file mode 100644 index 000000000..7d0769d13 --- /dev/null +++ b/zencore/snapshot_manifest.cpp @@ -0,0 +1,281 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <doctest/doctest.h> +#include <zencore/snapshot_manifest.h> +#include <zencore/stream.h> +#include <zencore/streamutil.h> +#include <zencore/string.h> +#include <ostream> + +#include <filesystem> + +#include <atlbase.h> + +// Used for getting My Documents for default snapshot dir +#include <ShlObj.h> +#pragma comment(lib, "shell32.lib") + +namespace zen { + +constexpr const char* magicString = "-=- ZEN_SNAP -=-"; + +struct SerializedManifestHeader +{ + char Magic[16]; + + void init() { memcpy(Magic, magicString, sizeof Magic); } + bool verify() const { return memcmp(Magic, magicString, sizeof Magic) == 0; } +}; + +TextWriter& +operator<<(TextWriter& Writer, const LeafNode& Leaf) +{ + Writer << "modTime: " << Leaf.FileModifiedTime << ", size: " << Leaf.FileSize << ", hash: " << Leaf.ChunkHash << ", name: " << Leaf.Name + << "\n"; + + return Writer; +} + +BinaryWriter& +operator<<(BinaryWriter& Writer, const LeafNode& Leaf) +{ + Writer << Leaf.FileModifiedTime << Leaf.FileSize << Leaf.ChunkHash << Leaf.Name; + + return Writer; +} + +BinaryReader& +operator>>(BinaryReader& Reader, LeafNode& Leaf) +{ + Reader >> Leaf.FileModifiedTime >> Leaf.FileSize >> Leaf.ChunkHash >> Leaf.Name; + + return Reader; +} + +void +TreeNode::Finalize() +{ + zen::BLAKE3Stream Blake3Stream; + + for (auto& Node : Children) + { + Node.Finalize(); + Blake3Stream.Append(Node.ChunkHash.Hash, sizeof Node.ChunkHash); + Blake3Stream.Append(Node.Name.data(), Node.Name.size() + 1); + } + + for (auto& leaf : Leaves) + { + Blake3Stream.Append(leaf.ChunkHash.Hash, sizeof leaf.ChunkHash); + Blake3Stream.Append(leaf.Name.data(), leaf.Name.size() + 1); + } + + this->ChunkHash = Blake3Stream.GetHash(); +} + +void +TreeNode::VisitFiles(std::function<void(const LeafNode& node)> func) +{ + for (auto& Node : Children) + Node.VisitFiles(func); + + for (auto& Leaf : Leaves) + func(Leaf); +} + +void +TreeNode::VisitModifyFiles(std::function<void(LeafNode& node)> func) +{ + for (auto& Node : Children) + Node.VisitModifyFiles(func); + + for (auto& Leaf : Leaves) + func(Leaf); +} + +IndentTextWriter& +operator<<(IndentTextWriter& Writer, const TreeNode& Node) +{ + Writer << "hash: " << Node.ChunkHash << ", name: " << Node.Name << "\n"; + + if (!Node.Leaves.empty()) + { + Writer << "files: " + << "\n"; + + IndentTextWriter::Scope _(Writer); + + for (const LeafNode& Leaf : Node.Leaves) + Writer << Leaf; + } + + if (!Node.Children.empty()) + { + Writer << "children: " + << "\n"; + + IndentTextWriter::Scope _(Writer); + + for (const TreeNode& Child : Node.Children) + { + Writer << Child; + } + } + + return Writer; +} + +BinaryWriter& +operator<<(BinaryWriter& Writer, const TreeNode& Node) +{ + Writer << Node.ChunkHash << Node.Name; + Writer << uint32_t(Node.Children.size()); + + for (const TreeNode& child : Node.Children) + Writer << child; + + Writer << uint32_t(Node.Leaves.size()); + + for (const LeafNode& Leaf : Node.Leaves) + Writer << Leaf; + + return Writer; +} + +BinaryReader& +operator>>(BinaryReader& Reader, TreeNode& Node) +{ + Reader >> Node.ChunkHash >> Node.Name; + + uint32_t ChildCount = 0; + Reader >> ChildCount; + Node.Children.resize(ChildCount); + + for (TreeNode& Child : Node.Children) + Reader >> Child; + + uint32_t LeafCount = 0; + Reader >> LeafCount; + Node.Leaves.resize(LeafCount); + + for (LeafNode& Leaf : Node.Leaves) + Reader >> Leaf; + + return Reader; +} + +void +SnapshotManifest::finalize() +{ + Root.Finalize(); + + zen::BLAKE3Stream Blake3Stream; + + Blake3Stream.Append(Root.ChunkHash.Hash, sizeof Root.ChunkHash); + Blake3Stream.Append(Root.Name.data(), Root.Name.size() + 1); + + this->ChunkHash = Blake3Stream.GetHash(); +} + +void +WriteManifest(const SnapshotManifest& Manifest, OutStream& ToStream) +{ + BinaryWriter Out(ToStream); + SerializedManifestHeader Header; + Header.init(); + Out.Write(&Header, sizeof Header); + + Out << Manifest.ChunkHash << Manifest.Id << Manifest.Root; +} + +void +ReadManifest(SnapshotManifest& Manifest, InStream& FromStream) +{ + BinaryReader Reader(FromStream); + SerializedManifestHeader Header; + Reader.Read(&Header, sizeof Header); + + Reader >> Manifest.ChunkHash >> Manifest.Id >> Manifest.Root; +} + +void +PrintManifest(const SnapshotManifest& Manifest, OutStream& ToStream) +{ + IndentTextWriter Writer(ToStream); + + Writer << "hash: " << Manifest.ChunkHash << "\n"; + Writer << "id: " << Manifest.Id << "\n"; + Writer << "root: " + << "\n"; + IndentTextWriter::Scope _(Writer); + Writer << Manifest.Root; +} + +std::filesystem::path +ManifestSpecToPath(const char* ManifestSpec) +{ + ExtendableWideStringBuilder<128> ManifestTargetFile; + + if (ManifestSpec[0] == '#') + { + // Pick sensible default + + WCHAR MyDocumentsDir[MAX_PATH]; + HRESULT hRes = SHGetFolderPathW(NULL, + CSIDL_PERSONAL /* My Documents */, + NULL, + SHGFP_TYPE_CURRENT, + /* out */ MyDocumentsDir); + + if (SUCCEEDED(hRes)) + { + wcscat_s(MyDocumentsDir, L"\\zenfs\\Snapshots\\"); + + ManifestTargetFile.Append(MyDocumentsDir); + ManifestTargetFile.AppendAscii(ManifestSpec + 1); + } + } + else + { + ManifestTargetFile.AppendAscii(ManifestSpec); + } + + std::filesystem::path ManifestPath{ManifestTargetFile.c_str()}; + + if (ManifestPath.extension() != L".zenfs") + { + ManifestPath.append(L".zenfs"); + } + + return ManifestPath; +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +void +snapshotmanifest_forcelink() +{ +} + +TEST_CASE("Snapshot manifest") +{ + SnapshotManifest Manifest; + + Manifest.Id = "test_manifest"; + Manifest.ChunkHash = zen::BLAKE3::HashMemory("abcd", 4); + + MemoryOutStream Outstream; + WriteManifest(Manifest, Outstream); + + MemoryInStream Instream(Outstream.Data(), Outstream.Size()); + SnapshotManifest Manifest2; + ReadManifest(/* out */ Manifest2, Instream); + + CHECK(Manifest.Id == Manifest2.Id); + CHECK(Manifest.ChunkHash == Manifest2.ChunkHash); +} + +} // namespace zen diff --git a/zencore/stats.cpp b/zencore/stats.cpp new file mode 100644 index 000000000..f8cdc8fbb --- /dev/null +++ b/zencore/stats.cpp @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/stats.h" +#include <doctest/doctest.h> +#include <cmath> +#include "zencore/timer.h" + +// +// Derived from https://github.com/dln/medida/blob/master/src/medida/stats/ewma.cc +// + +namespace zen { + +static constexpr int kTickInterval = 5; // In seconds +static constexpr double kSecondsPerMinute = 60.0; +static constexpr int kOneMinute = 1; +static constexpr int kFiveMinutes = 5; +static constexpr int kFifteenMinutes = 15; + +static double kM1_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kOneMinute); +static double kM5_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kFiveMinutes); +static double kM15_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kFifteenMinutes); + +static uint64_t CountPerTick = GetHifreqTimerFrequencySafe() * kTickInterval; +static uint64_t CountPerSecond = GetHifreqTimerFrequencySafe(); + +void +EWMA::Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate) +{ + double InstantRate = double(Count) / Interval; + + if (IsInitialUpdate) + { + m_rate = InstantRate; + } + else + { + m_rate += Alpha * (InstantRate - m_rate); + } +} + +double +EWMA::Rate() const +{ + return m_rate * CountPerSecond; +} + +////////////////////////////////////////////////////////////////////////// + +TEST_CASE("Stats") +{ + SUBCASE("Simple") + { + EWMA ewma1; + ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true); + + CHECK(ewma1.Rate() - 5 < 0.001); + + for (int i = 0; i < 60; ++i) + ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false); + + CHECK(ewma1.Rate() - 10 < 0.001); + + ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false); + } +} + +void +stats_forcelink() +{ +} + +} // namespace zen diff --git a/zencore/stream.cpp b/zencore/stream.cpp new file mode 100644 index 000000000..2cda0e123 --- /dev/null +++ b/zencore/stream.cpp @@ -0,0 +1,307 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <doctest/doctest.h> +#include <stdarg.h> +#include <zencore/memory.h> +#include <zencore/stream.h> +#include <algorithm> +#include <exception> + +namespace zen { + +MemoryInStream::MemoryInStream(const void* buffer, size_t size) +: m_Buffer(reinterpret_cast<const uint8_t*>(buffer), reinterpret_cast<const uint8_t*>(buffer) + size) +{ +} + +void +MemoryInStream::Read(void* buffer, size_t byteCount, uint64_t offset) +{ + RwLock::ExclusiveLockScope _(m_Lock); + + const size_t needEnd = offset + byteCount; + + if (needEnd > m_Buffer.size()) + throw std::exception("read past end of file!"); // TODO: better exception + + memcpy(buffer, m_Buffer.data() + offset, byteCount); +} + +void +MemoryOutStream::Write(const void* data, size_t byteCount, uint64_t offset) +{ + RwLock::ExclusiveLockScope _(m_Lock); + + const size_t needEnd = offset + byteCount; + + if (needEnd > m_Buffer.size()) + m_Buffer.resize(needEnd); + + memcpy(m_Buffer.data() + offset, data, byteCount); +} + +void +MemoryOutStream::Flush() +{ + // No-op +} + +////////////////////////////////////////////////////////////////////////// + +TextWriter::TextWriter(OutStream& stream) : m_Stream(&stream) +{ +} + +TextWriter::~TextWriter() = default; + +void +TextWriter::Write(const void* data, size_t byteCount) +{ + m_Stream->Write(data, byteCount, m_CurrentOffset); + m_CurrentOffset += byteCount; +} + +TextWriter& +operator<<(TextWriter& Writer, const char* value) +{ + if (value) + Writer.Write(value, strlen(value)); + else + Writer.Write("(null)", 6); + + return Writer; +} + +TextWriter& +operator<<(TextWriter& writer, const std::string_view& value) +{ + writer.Write(value.data(), value.size()); + + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, bool value) +{ + if (value) + writer.Write("true", 4); + else + writer.Write("false", 5); + + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, int8_t value) +{ + char buffer[16]; + _itoa_s(value, buffer, 10); + writer << buffer; + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, int16_t value) +{ + char buffer[16]; + _itoa_s(value, buffer, 10); + writer << buffer; + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, int32_t value) +{ + char buffer[16]; + _itoa_s(value, buffer, 10); + writer << buffer; + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, int64_t value) +{ + char buffer[32]; + _i64toa_s(value, buffer, sizeof buffer, 10); + writer << buffer; + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, uint8_t value) +{ + char buffer[16]; + _ultoa_s(value, buffer, 10); + writer << buffer; + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, uint16_t value) +{ + char buffer[16]; + _ultoa_s(value, buffer, 10); + writer << buffer; + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, uint32_t value) +{ + char buffer[16]; + _ultoa_s(value, buffer, 10); + writer << buffer; + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, uint64_t value) +{ + char buffer[32]; + _ui64toa_s(value, buffer, sizeof buffer, 10); + writer << buffer; + return writer; +} + +void +TextWriter::Writef(const char* formatString, ...) +{ + va_list args; + va_start(args, formatString); + + char* tempBuffer = nullptr; + char buffer[4096]; + int rv = vsnprintf(buffer, sizeof buffer, formatString, args); + + ZEN_ASSERT(rv >= 0); + + if (rv > sizeof buffer) + { + // Need more room -- allocate temporary buffer + + tempBuffer = (char*)Memory::Alloc(rv + 1, 8); + + int rv2 = vsnprintf(tempBuffer, rv + 1, formatString, args); + + ZEN_ASSERT(rv >= 0); + ZEN_ASSERT(rv2 <= rv); + + rv = rv2; + } + + m_Stream->Write(tempBuffer ? tempBuffer : buffer, rv, m_CurrentOffset); + m_CurrentOffset += rv; + + if (tempBuffer) + Memory::Free(tempBuffer); + + va_end(args); +} + +////////////////////////////////////////////////////////////////////////// + +IndentTextWriter::IndentTextWriter(OutStream& stream) : TextWriter(stream) +{ +} + +IndentTextWriter::~IndentTextWriter() +{ +} + +void +IndentTextWriter::Write(const void* data, size_t byteCount) +{ + const uint8_t* src = reinterpret_cast<const uint8_t*>(data); + int cur = m_LineCursor; + + while (byteCount) + { + char c = *src++; + + if (cur == 0) + { + const char indentSpaces[] = + " " + " "; + + cur = std::min<int>(m_IndentAmount, sizeof indentSpaces - 1); + memcpy(m_LineBuffer, indentSpaces, cur); + } + + m_LineBuffer[cur++] = c; + --byteCount; + + if (c == '\n' || cur == sizeof m_LineBuffer) + { + TextWriter::Write(m_LineBuffer, cur); + + cur = 0; + } + } + + m_LineCursor = cur; +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +void +stream_forcelink() +{ +} + +TEST_CASE("BinaryWriter and BinaryWriter") +{ + MemoryOutStream stream; + BinaryWriter writer(stream); + + CHECK(writer.CurrentOffset() == 0); + + writer.Write("foo!", 4); + CHECK(writer.CurrentOffset() == 4); + + writer << uint8_t(42) << uint16_t(42) << uint32_t(42) << uint64_t(42); + writer << int8_t(42) << int16_t(42) << int32_t(42) << int64_t(42); + + CHECK(writer.CurrentOffset() == (4 + 15 * 2)); + + // Read the data back + + MemoryInStream instream(stream.Data(), stream.Size()); + BinaryReader reader(instream); + CHECK(reader.CurrentOffset() == 0); + + char buffer[4]; + reader.Read(buffer, 4); + CHECK(reader.CurrentOffset() == 4); + + CHECK(memcmp(buffer, "foo!", 4) == 0); + + uint8_t ui8 = 0; + uint16_t ui16 = 0; + uint32_t ui32 = 0; + uint64_t ui64 = 0; + int8_t i8 = 0; + int16_t i16 = 0; + int32_t i32 = 0; + int64_t i64 = 0; + + reader >> ui8 >> ui16 >> ui32 >> ui64; + reader >> i8 >> i16 >> i32 >> i64; + + CHECK(reader.CurrentOffset() == (4 + 15 * 2)); + + CHECK(ui8 == 42); + CHECK(ui16 == 42); + CHECK(ui32 == 42); + CHECK(ui64 == 42); + + CHECK(i8 == 42); + CHECK(i16 == 42); + CHECK(i32 == 42); + CHECK(i64 == 42); +} + +} // namespace zen diff --git a/zencore/streamutil.cpp b/zencore/streamutil.cpp new file mode 100644 index 000000000..d3ed5ceaa --- /dev/null +++ b/zencore/streamutil.cpp @@ -0,0 +1,104 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/streamutil.h> +#include <zencore/string.h> + +namespace zen { + +BinaryWriter& +operator<<(BinaryWriter& writer, const std::string_view& value) +{ + writer.Write(value.data(), value.size()); + writer << uint8_t(0); + + return writer; +} + +BinaryReader& +operator>>(BinaryReader& reader, std::string& value) +{ + for (;;) + { + uint8_t x; + reader.Read(&x, 1); + + if (x == 0) + return reader; + + value.push_back(char(x)); + } +} + +BinaryWriter& +operator<<(BinaryWriter& writer, const std::wstring_view& value) +{ + // write as utf8 + + ExtendableStringBuilder<128> utf8; + WideToUtf8(value, utf8); + + writer.Write(utf8.c_str(), utf8.Size() + 1); + + return writer; +} + +BinaryReader& +operator>>(BinaryReader& reader, std::wstring& value) +{ + // read as utf8 + + std::string v8; + reader >> v8; + + ExtendableWideStringBuilder<128> wstr; + Utf8ToWide(v8, wstr); + + value = wstr.c_str(); + + return reader; +} + +TextWriter& +operator<<(TextWriter& writer, const zen::SHA1& value) +{ + zen::SHA1::String_t buffer; + value.ToHexString(buffer); + + writer.Write(buffer, zen::SHA1::StringLength); + + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, const zen::BLAKE3& value) +{ + zen::BLAKE3::String_t buffer; + value.ToHexString(buffer); + + writer.Write(buffer, zen::BLAKE3::StringLength); + + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, const zen::IoHash& value) +{ + zen::IoHash::String_t buffer; + value.ToHexString(buffer); + + writer.Write(buffer, zen::IoHash::StringLength); + + return writer; +} + +TextWriter& +operator<<(TextWriter& writer, const std::wstring_view& value) +{ + ExtendableStringBuilder<128> v8; + WideToUtf8(value, v8); + + writer.Write(v8.c_str(), v8.Size()); + return writer; +} + +} // namespace zen diff --git a/zencore/string.cpp b/zencore/string.cpp new file mode 100644 index 000000000..b6093ac2e --- /dev/null +++ b/zencore/string.cpp @@ -0,0 +1,913 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <doctest/doctest.h> +#include <inttypes.h> +#include <stdio.h> +#include <zencore/memory.h> +#include <zencore/string.h> +#include <exception> +#include <ostream> + +#include <utf8.h> + +template<typename u16bit_iterator> +void +utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuilderBase& OutString) +{ + while (StartIt != EndIt) + { + uint32_t cp = utf8::internal::mask16(*StartIt++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) + { + uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++); + cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET; + } + OutString.AppendCodepoint(cp); + } +} + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +bool +ToString(std::span<char> Buffer, uint64_t Num) +{ + snprintf(Buffer.data(), Buffer.size(), "%I64u", Num); + + return true; +} +bool +ToString(std::span<char> Buffer, int64_t Num) +{ + snprintf(Buffer.data(), Buffer.size(), "%I64d", Num); + + return true; +} + +////////////////////////////////////////////////////////////////////////// + +const char* +FilepathFindExtension(const std::string_view& Path, const char* ExtensionToMatch) +{ + const size_t PathLen = Path.size(); + + if (ExtensionToMatch) + { + size_t ExtLen = strlen(ExtensionToMatch); + + if (ExtLen > PathLen) + return nullptr; + + const char* PathExtension = Path.data() + PathLen - ExtLen; + + if (StringEquals(PathExtension, ExtensionToMatch)) + return PathExtension; + + return nullptr; + } + + if (PathLen == 0) + return nullptr; + + // Look for extension introducer ('.') + + for (size_t i = PathLen - 1; i >= 0; --i) + { + if (Path[i] == '.') + return Path.data() + i; + } + + return nullptr; +} + +////////////////////////////////////////////////////////////////////////// + +void +Utf8ToWide(const char8_t* Str8, WideStringBuilderBase& OutString) +{ + Utf8ToWide(std::u8string_view(Str8), OutString); +} + +void +Utf8ToWide(const std::string_view& Str8, WideStringBuilderBase& OutString) +{ + Utf8ToWide(std::u8string_view{reinterpret_cast<const char8_t*>(Str8.data()), Str8.size()}, OutString); +} + +std::wstring +Utf8ToWide(const std::string_view& Wstr) +{ + ExtendableWideStringBuilder<128> String; + Utf8ToWide(Wstr, String); + + return String.c_str(); +} + +void +Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString) +{ + const char* str = (const char*)Str8.data(); + const size_t strLen = Str8.size(); + + const char* endStr = str + strLen; + size_t ByteCount = 0; + size_t CurrentOutChar = 0; + + for (; str != endStr; ++str) + { + unsigned char Data = static_cast<unsigned char>(*str); + + if (!(Data & 0x80)) + { + // ASCII + OutString.Append(wchar_t(Data)); + continue; + } + else if (!ByteCount) + { + // Start of multi-byte sequence. Figure out how + // many bytes we're going to consume + + size_t Count = 0; + + for (size_t Temp = Data; Temp & 0x80; Temp <<= 1) + ++Count; + + ByteCount = Count - 1; + CurrentOutChar = Data & (0xff >> (Count + 1)); + } + else + { + --ByteCount; + + if ((Data & 0xc0) != 0x80) + { + break; + } + + CurrentOutChar = (CurrentOutChar << 6) | (Data & 0x3f); + + if (!ByteCount) + { + OutString.Append(wchar_t(CurrentOutChar)); + CurrentOutChar = 0; + } + } + } +} + +void +WideToUtf8(const wchar_t* Wstr, StringBuilderBase& OutString) +{ + WideToUtf8(std::u16string_view{(char16_t*)Wstr}, OutString); +} + +void +WideToUtf8(const std::wstring_view& Wstr, StringBuilderBase& OutString) +{ + WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, OutString); +} + +void +WideToUtf8(const std::u16string_view& Wstr, StringBuilderBase& OutString) +{ + utf16to8_impl(begin(Wstr), end(Wstr), OutString); +} + +std::string +WideToUtf8(const wchar_t* Wstr) +{ + ExtendableStringBuilder<128> String; + WideToUtf8(std::u16string_view{(char16_t*)Wstr}, String); + + return String.c_str(); +} + +std::string +WideToUtf8(const std::wstring_view Wstr) +{ + ExtendableStringBuilder<128> String; + WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, String); + + return String.c_str(); +} + +////////////////////////////////////////////////////////////////////////// + +enum NicenumFormat +{ + kNicenum1024 = 0, // Print kilo, mega, tera, peta, exa.. + kNicenumBytes = 1, // Print single bytes ("13B"), kilo, mega, tera... + kNicenumTime = 2, // Print nanosecs, microsecs, millisecs, seconds... + kNicenumRaw = 3, // Print the raw number without any formatting + kNicenumRawTime = 4 // Same as RAW, but print dashes ('-') for zero. +}; + +namespace { + static const char* UnitStrings[3][7] = { + /* kNicenum1024 */ {"", "K", "M", "G", "T", "P", "E"}, + /* kNicenumBytes */ {"B", "K", "M", "G", "T", "P", "E"}, + /* kNicenumTime */ {"ns", "us", "ms", "s", "?", "?", "?"}}; + + static const int UnitsLen[] = { + /* kNicenum1024 */ 6, + /* kNicenumBytes */ 6, + /* kNicenumTime */ 3}; + + static const uint64_t KiloUnit[] = { + /* kNicenum1024 */ 1024, + /* kNicenumBytes */ 1024, + /* kNicenumTime */ 1000}; +} // namespace + +/* + * Convert a number to an appropriately human-readable output. + */ +int +NiceNumGeneral(uint64_t Num, std::span<char> Buffer, NicenumFormat Format) +{ + switch (Format) + { + case kNicenumRaw: + return snprintf(Buffer.data(), Buffer.size(), "%llu", (uint64_t)Num); + + case kNicenumRawTime: + if (Num > 0) + { + return snprintf(Buffer.data(), Buffer.size(), "%llu", (uint64_t)Num); + } + else + { + return snprintf(Buffer.data(), Buffer.size(), "%s", "-"); + } + break; + + case kNicenum1024: + case kNicenumBytes: + case kNicenumTime: + default: + break; + } + + // Bring into range and select unit + + int Index = 0; + uint64_t n = Num; + + { + const uint64_t Unit = KiloUnit[Format]; + const int maxIndex = UnitsLen[Format]; + + while (n >= Unit && Index < maxIndex) + { + n /= Unit; + Index++; + } + } + + const char* u = UnitStrings[Format][Index]; + + if ((Index == 0) || ((Num % (uint64_t)powl((int)KiloUnit[Format], Index)) == 0)) + { + /* + * If this is an even multiple of the base, always display + * without any decimal precision. + */ + return snprintf(Buffer.data(), Buffer.size(), "%llu%s", (uint64_t)n, u); + } + else + { + /* + * We want to choose a precision that reflects the best choice + * for fitting in 5 characters. This can get rather tricky when + * we have numbers that are very close to an order of magnitude. + * For example, when displaying 10239 (which is really 9.999K), + * we want only a single place of precision for 10.0K. We could + * develop some complex heuristics for this, but it's much + * easier just to try each combination in turn. + */ + + int StrLen = 0; + + for (int i = 2; i >= 0; i--) + { + double Value = (double)Num / (uint64_t)powl((int)KiloUnit[Format], Index); + + /* + * Don't print floating point values for time. Note, + * we use floor() instead of round() here, since + * round can result in undesirable results. For + * example, if "num" is in the range of + * 999500-999999, it will print out "1000us". This + * doesn't happen if we use floor(). + */ + if (Format == kNicenumTime) + { + StrLen = snprintf(Buffer.data(), Buffer.size(), "%d%s", (unsigned int)floor(Value), u); + + if (StrLen <= 5) + break; + } + else + { + StrLen = snprintf(Buffer.data(), Buffer.size(), "%.*f%s", i, Value, u); + + if (StrLen <= 5) + break; + } + } + + return StrLen; + } +} + +size_t +NiceNumToBuffer(uint64_t Num, std::span<char> Buffer) +{ + return NiceNumGeneral(Num, Buffer, kNicenum1024); +} + +size_t +NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer) +{ + return NiceNumGeneral(Num, Buffer, kNicenumBytes); +} + +size_t +NiceByteRateToBuffer(uint64_t Num, uint64_t ElapsedMs, std::span<char> Buffer) +{ + size_t n = NiceNumGeneral(Num * 1000 / ElapsedMs, Buffer, kNicenumBytes); + + Buffer[n++] = '/'; + Buffer[n++] = 's'; + Buffer[n++] = '\0'; + + return n; +} + +size_t +NiceLatencyNsToBuffer(uint64_t Nanos, std::span<char> Buffer) +{ + return NiceNumGeneral(Nanos, Buffer, kNicenumTime); +} + +size_t +NiceTimeSpanMsToBuffer(uint64_t Millis, std::span<char> Buffer) +{ + if (Millis < 1000) + { + return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "ms", Millis); + } + else if (Millis < 10000) + { + return snprintf(Buffer.data(), Buffer.size(), "%.2fs", Millis / 1000.0); + } + else if (Millis < 60000) + { + return snprintf(Buffer.data(), Buffer.size(), "%.1fs", Millis / 1000.0); + } + else if (Millis < 60 * 60000) + { + return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "m%02" PRIu64 "s", Millis / 60000, (Millis / 1000) % 60); + } + else + { + return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "h%02" PRIu64 "m", Millis / 3600000, (Millis / 60000) % 60); + } +} + +////////////////////////////////////////////////////////////////////////// + +template<typename C> +StringBuilderImpl<C>::~StringBuilderImpl() +{ + if (m_IsDynamic) + { + FreeBuffer(m_Base, m_End - m_Base); + } +} + +template<typename C> +void +StringBuilderImpl<C>::Extend(size_t extraCapacity) +{ + if (!m_IsExtendable) + { + Fail("exceeded capacity"); + } + + const size_t oldCapacity = m_End - m_Base; + const size_t newCapacity = NextPow2(oldCapacity + extraCapacity); + + C* newBase = (C*)AllocBuffer(newCapacity); + + size_t pos = m_CurPos - m_Base; + memcpy(newBase, m_Base, pos * sizeof(C)); + + if (m_IsDynamic) + { + FreeBuffer(m_Base, oldCapacity); + } + + m_Base = newBase; + m_CurPos = newBase + pos; + m_End = newBase + newCapacity; + m_IsDynamic = true; +} + +template<typename C> +void* +StringBuilderImpl<C>::AllocBuffer(size_t byteCount) +{ + return Memory::Alloc(byteCount * sizeof(C)); +} + +template<typename C> +void +StringBuilderImpl<C>::FreeBuffer(void* buffer, size_t byteCount) +{ + ZEN_UNUSED(byteCount); + + Memory::Free(buffer); +} + +template<typename C> +[[noreturn]] void +StringBuilderImpl<C>::Fail(const char* reason) +{ + throw std::exception(reason); +} + +// Instantiate templates once + +template class StringBuilderImpl<char>; +template class StringBuilderImpl<wchar_t>; + +////////////////////////////////////////////////////////////////////////// +// +// Unit tests +// + +TEST_CASE("niceNum") +{ + char Buffer[16]; + + SUBCASE("raw") + { + NiceNumGeneral(1, Buffer, kNicenumRaw); + CHECK(StringEquals(Buffer, "1")); + + NiceNumGeneral(10, Buffer, kNicenumRaw); + CHECK(StringEquals(Buffer, "10")); + + NiceNumGeneral(100, Buffer, kNicenumRaw); + CHECK(StringEquals(Buffer, "100")); + + NiceNumGeneral(1000, Buffer, kNicenumRaw); + CHECK(StringEquals(Buffer, "1000")); + + NiceNumGeneral(10000, Buffer, kNicenumRaw); + CHECK(StringEquals(Buffer, "10000")); + + NiceNumGeneral(100000, Buffer, kNicenumRaw); + CHECK(StringEquals(Buffer, "100000")); + } + + SUBCASE("1024") + { + NiceNumGeneral(1, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1")); + + NiceNumGeneral(10, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "10")); + + NiceNumGeneral(100, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "100")); + + NiceNumGeneral(1000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1000")); + + NiceNumGeneral(10000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "9.77K")); + + NiceNumGeneral(100000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "97.7K")); + + NiceNumGeneral(1000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "977K")); + + NiceNumGeneral(10000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "9.54M")); + + NiceNumGeneral(100000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "95.4M")); + + NiceNumGeneral(1000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "954M")); + + NiceNumGeneral(10000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "9.31G")); + + NiceNumGeneral(100000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "93.1G")); + + NiceNumGeneral(1000000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "931G")); + + NiceNumGeneral(10000000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "9.09T")); + + NiceNumGeneral(100000000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "90.9T")); + + NiceNumGeneral(1000000000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "909T")); + + NiceNumGeneral(10000000000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "8.88P")); + + NiceNumGeneral(100000000000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "88.8P")); + + NiceNumGeneral(1000000000000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "888P")); + + NiceNumGeneral(10000000000000000000, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "8.67E")); + + // pow2 + + NiceNumGeneral(0, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "0")); + + NiceNumGeneral(1, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1")); + + NiceNumGeneral(1024, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1K")); + + NiceNumGeneral(1024 * 1024, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1M")); + + NiceNumGeneral(1024 * 1024 * 1024, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1G")); + + NiceNumGeneral(1024llu * 1024 * 1024 * 1024, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1T")); + + NiceNumGeneral(1024llu * 1024 * 1024 * 1024 * 1024, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1P")); + + NiceNumGeneral(1024llu * 1024 * 1024 * 1024 * 1024 * 1024, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1E")); + + // pow2-1 + + NiceNumGeneral(1023, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "1023")); + + NiceNumGeneral(2047, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "2.00K")); + + NiceNumGeneral(9 * 1024 - 1, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "9.00K")); + + NiceNumGeneral(10 * 1024 - 1, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "10.0K")); + + NiceNumGeneral(10 * 1024 - 5, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "10.0K")); + + NiceNumGeneral(10 * 1024 - 6, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "9.99K")); + + NiceNumGeneral(10 * 1024 - 10, Buffer, kNicenum1024); + CHECK(StringEquals(Buffer, "9.99K")); + } + + SUBCASE("time") + { + NiceNumGeneral(1, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "1ns")); + + NiceNumGeneral(100, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "100ns")); + + NiceNumGeneral(1000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "1us")); + + NiceNumGeneral(10000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "10us")); + + NiceNumGeneral(100000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "100us")); + + NiceNumGeneral(1000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "1ms")); + + NiceNumGeneral(10000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "10ms")); + + NiceNumGeneral(100000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "100ms")); + + NiceNumGeneral(1000000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "1s")); + + NiceNumGeneral(10000000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "10s")); + + NiceNumGeneral(100000000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "100s")); + + NiceNumGeneral(1000000000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "1000s")); + + NiceNumGeneral(10000000000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "10000s")); + + NiceNumGeneral(100000000000000, Buffer, kNicenumTime); + CHECK(StringEquals(Buffer, "100000s")); + } + + SUBCASE("bytes") + { + NiceNumGeneral(1, Buffer, kNicenumBytes); + CHECK(StringEquals(Buffer, "1B")); + + NiceNumGeneral(10, Buffer, kNicenumBytes); + CHECK(StringEquals(Buffer, "10B")); + + NiceNumGeneral(100, Buffer, kNicenumBytes); + CHECK(StringEquals(Buffer, "100B")); + + NiceNumGeneral(1000, Buffer, kNicenumBytes); + CHECK(StringEquals(Buffer, "1000B")); + + NiceNumGeneral(10000, Buffer, kNicenumBytes); + CHECK(StringEquals(Buffer, "9.77K")); + } + + SUBCASE("byteRate") + { + NiceByteRateToBuffer(1, 1, Buffer); + CHECK(StringEquals(Buffer, "1000B/s")); + + NiceByteRateToBuffer(1000, 1000, Buffer); + CHECK(StringEquals(Buffer, "1000B/s")); + + NiceByteRateToBuffer(1024, 1, Buffer); + CHECK(StringEquals(Buffer, "1000K/s")); + + NiceByteRateToBuffer(1024, 1000, Buffer); + CHECK(StringEquals(Buffer, "1K/s")); + } + + SUBCASE("timespan") + { + NiceTimeSpanMsToBuffer(1, Buffer); + CHECK(StringEquals(Buffer, "1ms")); + + NiceTimeSpanMsToBuffer(900, Buffer); + CHECK(StringEquals(Buffer, "900ms")); + + NiceTimeSpanMsToBuffer(1000, Buffer); + CHECK(StringEquals(Buffer, "1.00s")); + + NiceTimeSpanMsToBuffer(1900, Buffer); + CHECK(StringEquals(Buffer, "1.90s")); + + NiceTimeSpanMsToBuffer(19000, Buffer); + CHECK(StringEquals(Buffer, "19.0s")); + + NiceTimeSpanMsToBuffer(60000, Buffer); + CHECK(StringEquals(Buffer, "1m00s")); + + NiceTimeSpanMsToBuffer(600000, Buffer); + CHECK(StringEquals(Buffer, "10m00s")); + + NiceTimeSpanMsToBuffer(3600000, Buffer); + CHECK(StringEquals(Buffer, "1h00m")); + + NiceTimeSpanMsToBuffer(36000000, Buffer); + CHECK(StringEquals(Buffer, "10h00m")); + + NiceTimeSpanMsToBuffer(360000000, Buffer); + CHECK(StringEquals(Buffer, "100h00m")); + } +} + +void +string_forcelink() +{ +} + +TEST_CASE("StringBuilder") +{ + StringBuilder<64> sb; + + SUBCASE("Empty init") + { + const char* str = sb.c_str(); + + CHECK(StringLength(str) == 0); + } + + SUBCASE("Append single character") + { + sb.Append('a'); + + const char* str = sb.c_str(); + CHECK(StringLength(str) == 1); + CHECK(str[0] == 'a'); + + sb.Append('b'); + str = sb.c_str(); + CHECK(StringLength(str) == 2); + CHECK(str[0] == 'a'); + CHECK(str[1] == 'b'); + } + + SUBCASE("Append string") + { + sb.Append("a"); + + const char* str = sb.c_str(); + CHECK(StringLength(str) == 1); + CHECK(str[0] == 'a'); + + sb.Append("b"); + str = sb.c_str(); + CHECK(StringLength(str) == 2); + CHECK(str[0] == 'a'); + CHECK(str[1] == 'b'); + + sb.Append("cdefghijklmnopqrstuvwxyz"); + CHECK(sb.Size() == 26); + + sb.Append("abcdefghijklmnopqrstuvwxyz"); + CHECK(sb.Size() == 52); + + sb.Append("abcdefghijk"); + CHECK(sb.Size() == 63); + } +} + +TEST_CASE("ExtendableStringBuilder") +{ + ExtendableStringBuilder<16> sb; + + SUBCASE("Empty init") + { + const char* str = sb.c_str(); + + CHECK(StringLength(str) == 0); + } + + SUBCASE("Short append") + { + sb.Append("abcd"); + CHECK(sb.IsDynamic() == false); + } + + SUBCASE("Short+long append") + { + sb.Append("abcd"); + CHECK(sb.IsDynamic() == false); + // This should trigger a dynamic buffer allocation since the required + // capacity exceeds the internal fixed buffer. + sb.Append("abcdefghijklmnopqrstuvwxyz"); + CHECK(sb.IsDynamic() == true); + CHECK(sb.Size() == 30); + CHECK(sb.Size() == StringLength(sb.c_str())); + } +} + +TEST_CASE("WideStringBuilder") +{ + WideStringBuilder<64> sb; + + SUBCASE("Empty init") + { + const wchar_t* str = sb.c_str(); + + CHECK(StringLength(str) == 0); + } + + SUBCASE("Append single character") + { + sb.Append(L'a'); + + const wchar_t* str = sb.c_str(); + CHECK(StringLength(str) == 1); + CHECK(str[0] == L'a'); + + sb.Append(L'b'); + str = sb.c_str(); + CHECK(StringLength(str) == 2); + CHECK(str[0] == L'a'); + CHECK(str[1] == L'b'); + } + + SUBCASE("Append string") + { + sb.Append(L"a"); + + const wchar_t* str = sb.c_str(); + CHECK(StringLength(str) == 1); + CHECK(str[0] == L'a'); + + sb.Append(L"b"); + str = sb.c_str(); + CHECK(StringLength(str) == 2); + CHECK(str[0] == L'a'); + CHECK(str[1] == L'b'); + + sb.Append(L"cdefghijklmnopqrstuvwxyz"); + CHECK(sb.Size() == 26); + + sb.Append(L"abcdefghijklmnopqrstuvwxyz"); + CHECK(sb.Size() == 52); + + sb.Append(L"abcdefghijk"); + CHECK(sb.Size() == 63); + } +} + +TEST_CASE("ExtendableWideStringBuilder") +{ + ExtendableWideStringBuilder<16> sb; + + SUBCASE("Empty init") + { + CHECK(sb.Size() == 0); + + const wchar_t* str = sb.c_str(); + CHECK(StringLength(str) == 0); + } + + SUBCASE("Short append") + { + sb.Append(L"abcd"); + CHECK(sb.IsDynamic() == false); + } + + SUBCASE("Short+long append") + { + sb.Append(L"abcd"); + CHECK(sb.IsDynamic() == false); + // This should trigger a dynamic buffer allocation since the required + // capacity exceeds the internal fixed buffer. + sb.Append(L"abcdefghijklmnopqrstuvwxyz"); + CHECK(sb.IsDynamic() == true); + CHECK(sb.Size() == 30); + CHECK(sb.Size() == StringLength(sb.c_str())); + } +} + +TEST_CASE("utf8") +{ + SUBCASE("utf8towide") + { + // TODO: add more extensive testing here - this covers a very small space + + WideStringBuilder<32> wout; + Utf8ToWide(u8"abcdefghi", wout); + CHECK(StringEquals(L"abcdefghi", wout.c_str())); + + wout.Reset(); + + Utf8ToWide(u8"abc���", wout); + CHECK(StringEquals(L"abc���", wout.c_str())); + } + + SUBCASE("widetoutf8") + { + // TODO: add more extensive testing here - this covers a very small space + + StringBuilder<32> out; + + WideToUtf8(L"abcdefghi", out); + CHECK(StringEquals("abcdefghi", out.c_str())); + + out.Reset(); + + WideToUtf8(L"abc���", out); + CHECK(StringEquals(u8"abc���", out.c_str())); + } +} + +TEST_CASE("filepath") +{ + CHECK(FilepathFindExtension("foo\\bar\\baz.txt", ".txt") != nullptr); + CHECK(FilepathFindExtension("foo\\bar\\baz.txt", ".zap") == nullptr); + + CHECK(FilepathFindExtension("foo\\bar\\baz.txt") != nullptr); + CHECK(FilepathFindExtension("foo\\bar\\baz.txt") == std::string_view(".txt")); + + CHECK(FilepathFindExtension(".txt") == std::string_view(".txt")); +} + +} // namespace zen diff --git a/zencore/thread.cpp b/zencore/thread.cpp new file mode 100644 index 000000000..80cf6f100 --- /dev/null +++ b/zencore/thread.cpp @@ -0,0 +1,192 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/thread.h> + +#include <zencore/except.h> +#include <zencore/string.h> +#include <zencore/windows.h> +#include <thread> + +namespace zen { + +void +RwLock::AcquireShared() +{ + AcquireSRWLockShared((PSRWLOCK)&m_Srw); +} + +void +RwLock::ReleaseShared() +{ + ReleaseSRWLockShared((PSRWLOCK)&m_Srw); +} + +void +RwLock::AcquireExclusive() +{ + AcquireSRWLockExclusive((PSRWLOCK)&m_Srw); +} + +void +RwLock::ReleaseExclusive() +{ + ReleaseSRWLockExclusive((PSRWLOCK)&m_Srw); +} + +Event::Event() +{ + m_EventHandle = CreateEvent(nullptr, true, false, nullptr); +} + +Event::~Event() +{ + CloseHandle(m_EventHandle); +} + +void +Event::Set() +{ + SetEvent(m_EventHandle); +} + +void +Event::Reset() +{ + ResetEvent(m_EventHandle); +} + +bool +Event::Wait(int TimeoutMs) +{ + const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs; + + DWORD Result = WaitForSingleObject(m_EventHandle, Timeout); + + if (Result == WAIT_FAILED) + { + throw WindowsException("Event wait failed"); + } + + return (Result == WAIT_OBJECT_0); +} + +NamedEvent::NamedEvent(std::u8string_view EventName) : Event(nullptr) +{ + using namespace std::literals; + + ExtendableStringBuilder<64> Name; + Name << "Local\\"sv; + Name << EventName; + + m_EventHandle = CreateEventA(nullptr, true, false, Name.c_str()); +} + +NamedEvent::NamedEvent(std::string_view EventName) : Event(nullptr) +{ + using namespace std::literals; + + ExtendableStringBuilder<64> Name; + Name << "Local\\"sv; + Name << EventName; + + m_EventHandle = CreateEventA(nullptr, true, false, Name.c_str()); +} + +Process::Process() = default; + +void +Process::Initialize(void* ProcessHandle) +{ + ZEN_ASSERT(m_ProcessHandle == nullptr); + // TODO: perform some debug verification here to verify it's a valid handle? + m_ProcessHandle = ProcessHandle; +} + +Process::~Process() +{ + if (IsValid()) + { + CloseHandle(m_ProcessHandle); + m_ProcessHandle = nullptr; + } +} + +void +Process::Initialize(int Pid) +{ + ZEN_ASSERT(m_ProcessHandle == nullptr); + m_ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, Pid); + m_Pid = Pid; +} + +bool +Process::IsRunning() const +{ + DWORD ExitCode = 0; + GetExitCodeProcess(m_ProcessHandle, &ExitCode); + + return ExitCode == STILL_ACTIVE; +} + +bool +Process::IsValid() const +{ + return (m_ProcessHandle != nullptr) && (m_ProcessHandle != INVALID_HANDLE_VALUE); +} + +void +Process::Terminate(int ExitCode) +{ + if (IsRunning()) + { + TerminateProcess(m_ProcessHandle, ExitCode); + } + + DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, INFINITE); + + if (WaitResult != WAIT_OBJECT_0) + { + // What might go wrong here, and what is meaningful to act on? + } +} + +bool +Process::Wait(int TimeoutMs) +{ + const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs; + + const DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, Timeout); + + switch (WaitResult) + { + case WAIT_OBJECT_0: + return true; + + case WAIT_TIMEOUT: + return false; + + case WAIT_FAILED: + // What might go wrong here, and what is meaningful to act on? + throw WindowsException("Process::Wait failed"); + } + + return false; +} + +void +Sleep(int ms) +{ + ::Sleep(ms); +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +void +thread_forcelink() +{ +} + +} // namespace zen diff --git a/zencore/timer.cpp b/zencore/timer.cpp new file mode 100644 index 000000000..ee8e1cf9c --- /dev/null +++ b/zencore/timer.cpp @@ -0,0 +1,67 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <doctest/doctest.h> +#include <zencore/thread.h> +#include <zencore/timer.h> +#include <zencore/windows.h> + +namespace zen { + +uint64_t +GetHifreqTimerValue() +{ + LARGE_INTEGER li; + QueryPerformanceCounter(&li); + + return li.QuadPart; +} + +uint64_t +internalGetHifreqTimerFrequency() +{ + LARGE_INTEGER li; + QueryPerformanceFrequency(&li); + + return li.QuadPart; +} + +static uint64_t qpcFreq = internalGetHifreqTimerFrequency(); + +uint64_t +GetHifreqTimerFrequency() +{ + return qpcFreq; +} + +uint64_t +GetHifreqTimerFrequencySafe() +{ + if (!qpcFreq) + qpcFreq = internalGetHifreqTimerFrequency(); + + return qpcFreq; +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +void +timer_forcelink() +{ +} + +TEST_CASE("Timer") +{ + uint64_t s0 = GetHifreqTimerValue(); + uint64_t t0 = GetCpuTimerValue(); + Sleep(1000); + uint64_t s1 = GetHifreqTimerValue(); + uint64_t t1 = GetCpuTimerValue(); + // double r = double(t1 - t0) / (s1 - s0); + CHECK_NE(t0, t1); + CHECK_NE(s0, s1); +} + +} // namespace zen diff --git a/zencore/trace.cpp b/zencore/trace.cpp new file mode 100644 index 000000000..8313b3b66 --- /dev/null +++ b/zencore/trace.cpp @@ -0,0 +1,51 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/trace.h" +#include <doctest/doctest.h> +#include <zencore/windows.h> + +namespace zen { + +void +Tracer::Log(const TraceEvent& e) +{ + TraceBroadcast(e); +} + +Tracer g_globalTracer; + +struct alignas(64) TraceHandlerList +{ + enum + { + kMaxHandlers = 7 + }; + + uint8_t handlerCount = 0; + TraceHandler* handlers[kMaxHandlers]; +}; + +static TraceHandlerList g_traceHandlers; + +void +TraceBroadcast(const TraceEvent& e) +{ + for (size_t i = 0; i < g_traceHandlers.handlerCount; ++i) + { + g_traceHandlers.handlers[i]->Trace(e); + } +} + +void +trace_forcelink() +{ +} + +////////////////////////////////////////////////////////////////////////// + +TEST_CASE("Tracer") +{ + SUBCASE("Simple") { U_LOG_INFO("bajs"); } +} + +} // namespace zen diff --git a/zencore/uid.cpp b/zencore/uid.cpp new file mode 100644 index 000000000..9506b305c --- /dev/null +++ b/zencore/uid.cpp @@ -0,0 +1,196 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/uid.h> + +#include <zencore/string.h> + +#if _WIN32 +# include <zencore/windows.h> +# include <bcrypt.h> +# pragma comment(lib, "bcrypt.lib") +#endif + +#include <atomic> +#include <bit> +#include <set> +#include <unordered_map> + +#include <doctest/doctest.h> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +template<typename T> +T +EndianSwap(T value) +{ + uint8_t dest[sizeof value]; + + memcpy(dest, &value, sizeof value); + + for (int i = 0; i < sizeof(value); i++) + { + uint8_t& other = dest[sizeof(value) - i - 1]; + + uint8_t temp = dest[i]; + dest[i] = other; + other = temp; + } + + T ret; + + memcpy(&ret, &value, sizeof value); + + return ret; +} + +#if _WIN32 +__forceinline uint16_t +EndianSwap(uint16_t value) +{ + return _byteswap_ushort(value); +} + +__forceinline uint32_t +EndianSwap(uint32_t value) +{ + return _byteswap_ulong(value); +} + +__forceinline uint64_t +EndianSwap(uint64_t value) +{ + return _byteswap_uint64(value); +} +#endif + +////////////////////////////////////////////////////////////////////////// + +namespace detail { + static bool OidInitialised; + static uint32_t RunId; + static std::atomic_uint32_t Serial; + + // Number of 100 nanosecond units from 1/1/1601 to 1/1/1970 - used for Windows impl + constexpr int64_t kEpochBias = 116'444'736'000'000'000ull; +} // namespace detail + +////////////////////////////////////////////////////////////////////////// + +const Oid Oid::Zero = {{0u, 0u, 0u}}; +const Oid Oid::Max = {{~0u, ~0u, ~0u}}; + +void +Oid::Initialize() +{ + using namespace detail; + + if (OidInitialised) + return; + + OidInitialised = true; + +#if _WIN32 + char rng[8]; + BCryptGenRandom(NULL, (PUCHAR)rng, sizeof rng, BCRYPT_USE_SYSTEM_PREFERRED_RNG); + + memcpy(&RunId, &rng[0], sizeof(RunId)); + memcpy((void*)&Serial, &rng[4], sizeof(Serial)); +#else +# error Must implement Oid::Initialize +#endif +} + +const Oid& +Oid::Generate() +{ + using namespace detail; + + if (!OidInitialised) + { + Oid::Initialize(); + } + +#if _WIN32 + FILETIME filetime; + + GetSystemTimeAsFileTime(&filetime); // Time is UTC + + uint64_t filetime64; + memcpy(&filetime64, &filetime, sizeof filetime); + + OidBits[0] = EndianSwap(uint32_t((filetime64 - kEpochBias) / 10'000'000l)); + OidBits[1] = EndianSwap(uint32_t(Serial++)); + OidBits[2] = RunId; +#else +# error Must implement Oid::Generate +#endif + + return *this; +} + +Oid +Oid::NewOid() +{ + return Oid().Generate(); +} + +Oid +Oid::FromHexString(const std::string_view String) +{ + ZEN_ASSERT(String.size() == 2 * sizeof(Oid::OidBits)); + + Oid Id; + + ParseHexBytes(String.data(), String.size(), reinterpret_cast<uint8_t*>(Id.OidBits)); + + return Id; +} + +StringBuilderBase& +Oid::ToString(StringBuilderBase& OutString) const +{ + char str[25]; + ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), str); + str[2 * sizeof(Oid)] = '\0'; + + OutString.AppendRange(str, &str[25]); + + return OutString; +} + +TEST_CASE("Oid") +{ + SUBCASE("Basic") + { + Oid id1 = Oid::NewOid(); + + std::vector<Oid> ids; + std::set<Oid> idset; + std::unordered_map<Oid, int, Oid::Hasher> idmap; + + const int Count = 1000; + + for (int i = 0; i < Count; ++i) + { + Oid id; + id.Generate(); + + ids.emplace_back(id); + idset.insert(id); + idmap.insert({id, i}); + } + + CHECK(ids.size() == Count); + CHECK(idset.size() == Count); // All ids should be unique + CHECK(idmap.size() == Count); // Ditto + } +} + +void +uid_forcelink() +{ +} + +} // namespace zen diff --git a/zencore/xxhash.cpp b/zencore/xxhash.cpp new file mode 100644 index 000000000..a20ee10bd --- /dev/null +++ b/zencore/xxhash.cpp @@ -0,0 +1,50 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/xxhash.h> + +#include <zencore/string.h> + +#include <doctest/doctest.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +XXH3_128 XXH3_128::Zero; // Initialized to all zeros + +XXH3_128 +XXH3_128::FromHexString(const char* InString) +{ + return FromHexString({InString, sizeof(XXH3_128::Hash) * 2}); +} + +XXH3_128 +XXH3_128::FromHexString(std::string_view InString) +{ + ZEN_ASSERT(InString.size() == 2 * sizeof(XXH3_128::Hash)); + + XXH3_128 Xx; + ParseHexBytes(InString.data(), InString.size(), Xx.Hash); + return Xx; +} + +const char* +XXH3_128::ToHexString(char* OutString /* 40 characters + NUL terminator */) const +{ + ToHexBytes(Hash, sizeof(XXH3_128), OutString); + OutString[2 * sizeof(XXH3_128)] = '\0'; + + return OutString; +} + +StringBuilderBase& +XXH3_128::ToHexString(StringBuilderBase& OutBuilder) const +{ + String_t str; + ToHexString(str); + + OutBuilder.AppendRange(str, &str[StringLength]); + + return OutBuilder; +} + +} // namespace zen diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp new file mode 100644 index 000000000..d4853b043 --- /dev/null +++ b/zencore/zencore.cpp @@ -0,0 +1,70 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/zencore.h> + +#include <zencore/windows.h> + +#include <zencore/blake3.h> +#include <zencore/compactbinary.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/iobuffer.h> +#include <zencore/memory.h> +#include <zencore/refcount.h> +#include <zencore/sha1.h> +#include <zencore/snapshot_manifest.h> +#include <zencore/stats.h> +#include <zencore/stream.h> +#include <zencore/string.h> +#include <zencore/thread.h> +#include <zencore/timer.h> +#include <zencore/trace.h> +#include <zencore/uid.h> + +bool +IsPointerToStack(const void* ptr) +{ + ULONG_PTR low, high; + GetCurrentThreadStackLimits(&low, &high); + + const uintptr_t intPtr = reinterpret_cast<uintptr_t>(ptr); + + return (intPtr - low) < (high - low); +} + +static int s_ApplicationExitCode = 0; +static bool s_ApplicationExitRequested; + +bool +IsApplicationExitRequested() +{ + return s_ApplicationExitRequested; +} + +void +RequestApplicationExit(int ExitCode) +{ + s_ApplicationExitCode = ExitCode; + s_ApplicationExitRequested = true; +} + +void +zencore_forcelinktests() +{ + zen::sha1_forcelink(); + zen::blake3_forcelink(); + zen::trace_forcelink(); + zen::timer_forcelink(); + zen::uid_forcelink(); + zen::string_forcelink(); + zen::thread_forcelink(); + zen::stream_forcelink(); + zen::refcount_forcelink(); + zen::snapshotmanifest_forcelink(); + zen::iobuffer_forcelink(); + zen::stats_forcelink(); + zen::uson_forcelink(); + zen::usonbuilder_forcelink(); + zen::usonpackage_forcelink(); + zen::memory_forcelink(); +} diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj new file mode 100644 index 000000000..c68e922c5 --- /dev/null +++ b/zencore/zencore.vcxproj @@ -0,0 +1,188 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <VCProjectVersion>15.0</VCProjectVersion> + <ProjectGuid>{D75BF9AB-C61E-4FFF-AD59-1563430F05E2}</ProjectGuid> + <Keyword>Win32Proj</Keyword> + <RootNamespace>zencore</RootNamespace> + <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> + <ConfigurationType>StaticLibrary</ConfigurationType> + <UseDebugLibraries>true</UseDebugLibraries> + <PlatformToolset>v142</PlatformToolset> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> + <ConfigurationType>StaticLibrary</ConfigurationType> + <UseDebugLibraries>false</UseDebugLibraries> + <PlatformToolset>v142</PlatformToolset> + <WholeProgramOptimization>false</WholeProgramOptimization> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="Shared"> + </ImportGroup> + <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="..\zenfs_common.props" /> + <Import Project="..\zen_base_debug.props" /> + </ImportGroup> + <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="..\zenfs_common.props" /> + <Import Project="..\zen_base_release.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <LinkIncremental>true</LinkIncremental> + <EnableMicrosoftCodeAnalysis>false</EnableMicrosoftCodeAnalysis> + <EnableClangTidyCodeAnalysis>true</EnableClangTidyCodeAnalysis> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <LinkIncremental>false</LinkIncremental> + </PropertyGroup> + <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <VcpkgEnableManifest>true</VcpkgEnableManifest> + <VcpkgUseStatic>true</VcpkgUseStatic> + </PropertyGroup> + <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <VcpkgEnableManifest>true</VcpkgEnableManifest> + <VcpkgUseStatic>true</VcpkgUseStatic> + </PropertyGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <Optimization>Disabled</Optimization> + <SDLCheck>true</SDLCheck> + <PreprocessorDefinitions>_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <ConformanceMode>true</ConformanceMode> + <AdditionalIncludeDirectories>.\include;..\3rdparty\utfcpp\source</AdditionalIncludeDirectories> + <DebugInformationFormat>ProgramDatabase</DebugInformationFormat> + <LanguageStandard>stdcpplatest</LanguageStandard> + <TreatWarningAsError>true</TreatWarningAsError> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + </ClCompile> + <Link> + <SubSystem>Windows</SubSystem> + <GenerateDebugInformation>true</GenerateDebugInformation> + </Link> + <Lib /> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <Optimization>MaxSpeed</Optimization> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <PreprocessorDefinitions>NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <ConformanceMode>true</ConformanceMode> + <AdditionalIncludeDirectories>.\include;..\3rdparty\utfcpp\source</AdditionalIncludeDirectories> + <WholeProgramOptimization>false</WholeProgramOptimization> + <LanguageStandard>stdcpplatest</LanguageStandard> + <TreatWarningAsError>true</TreatWarningAsError> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + </ClCompile> + <Link> + <SubSystem>Windows</SubSystem> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + <GenerateDebugInformation>true</GenerateDebugInformation> + </Link> + <Lib> + <Verbose>true</Verbose> + </Lib> + </ItemDefinitionGroup> + <ItemGroup> + <ClInclude Include="include\zencore\atomic.h" /> + <ClInclude Include="include\zencore\blake3.h" /> + <ClInclude Include="include\zencore\enumflags.h" /> + <ClInclude Include="include\zencore\except.h" /> + <ClInclude Include="include\zencore\compress.h" /> + <ClInclude Include="include\zencore\filesystem.h" /> + <ClInclude Include="include\zencore\fmtutils.h" /> + <ClInclude Include="include\zencore\httpclient.h" /> + <ClInclude Include="include\zencore\httpserver.h" /> + <ClInclude Include="include\zencore\intmath.h" /> + <ClInclude Include="include\zencore\iohash.h" /> + <ClInclude Include="include\zencore\md5.h" /> + <ClInclude Include="include\zencore\memory.h" /> + <ClInclude Include="include\zencore\refcount.h" /> + <ClInclude Include="include\zencore\scopeguard.h" /> + <ClInclude Include="include\zencore\sha1.h" /> + <ClInclude Include="include\zencore\iobuffer.h" /> + <ClInclude Include="include\zencore\sharedbuffer.h" /> + <ClInclude Include="include\zencore\snapshot_manifest.h" /> + <ClInclude Include="include\zencore\stats.h" /> + <ClInclude Include="include\zencore\stream.h" /> + <ClInclude Include="include\zencore\streamutil.h" /> + <ClInclude Include="include\zencore\string.h" /> + <ClInclude Include="include\zencore\targetver.h" /> + <ClInclude Include="include\zencore\thread.h" /> + <ClInclude Include="include\zencore\timer.h" /> + <ClInclude Include="include\zencore\trace.h" /> + <ClInclude Include="include\zencore\uid.h" /> + <ClInclude Include="include\zencore\compactbinary.h" /> + <ClInclude Include="include\zencore\compactbinarybuilder.h" /> + <ClInclude Include="include\zencore\compactbinarypackage.h" /> + <ClInclude Include="include\zencore\compactbinaryvalidation.h" /> + <ClInclude Include="include\zencore\varint.h" /> + <ClInclude Include="include\zencore\windows.h" /> + <ClInclude Include="include\zencore\xxhash.h" /> + <ClInclude Include="include\zencore\zencore.h" /> + <ClInclude Include="iothreadpool.h" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="blake3.cpp" /> + <ClCompile Include="compress.cpp" /> + <ClCompile Include="except.cpp" /> + <ClCompile Include="filesystem.cpp" /> + <ClCompile Include="httpclient.cpp" /> + <ClCompile Include="httpserver.cpp" /> + <ClCompile Include="iohash.cpp" /> + <ClCompile Include="iothreadpool.cpp" /> + <ClCompile Include="md5.cpp" /> + <ClCompile Include="memory.cpp" /> + <ClCompile Include="refcount.cpp" /> + <ClCompile Include="sha1.cpp"> + <Optimization Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">MaxSpeed</Optimization> + <InlineFunctionExpansion Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AnySuitable</InlineFunctionExpansion> + <IntrinsicFunctions Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</IntrinsicFunctions> + <BasicRuntimeChecks Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Default</BasicRuntimeChecks> + </ClCompile> + <ClCompile Include="iobuffer.cpp" /> + <ClCompile Include="sharedbuffer.cpp" /> + <ClCompile Include="snapshot_manifest.cpp" /> + <ClCompile Include="stats.cpp" /> + <ClCompile Include="stream.cpp" /> + <ClCompile Include="streamutil.cpp" /> + <ClCompile Include="string.cpp" /> + <ClCompile Include="thread.cpp" /> + <ClCompile Include="timer.cpp" /> + <ClCompile Include="trace.cpp" /> + <ClCompile Include="uid.cpp" /> + <ClCompile Include="compactbinary.cpp" /> + <ClCompile Include="compactbinarybuilder.cpp" /> + <ClCompile Include="compactbinarypackage.cpp" /> + <ClCompile Include="compactbinaryvalidation.cpp" /> + <ClCompile Include="xxhash.cpp" /> + <ClCompile Include="zencore.cpp" /> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + </ImportGroup> +</Project>
\ No newline at end of file diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters new file mode 100644 index 000000000..c25f99e77 --- /dev/null +++ b/zencore/zencore.vcxproj.filters @@ -0,0 +1,78 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <ClInclude Include="include\zencore\intmath.h" /> + <ClInclude Include="include\zencore\scopeguard.h" /> + <ClInclude Include="include\zencore\sha1.h" /> + <ClInclude Include="include\zencore\snapshot_manifest.h" /> + <ClInclude Include="include\zencore\targetver.h" /> + <ClInclude Include="include\zencore\zencore.h" /> + <ClInclude Include="include\zencore\compactbinary.h" /> + <ClInclude Include="include\zencore\uid.h" /> + <ClInclude Include="include\zencore\trace.h" /> + <ClInclude Include="include\zencore\compress.h" /> + <ClInclude Include="include\zencore\timer.h" /> + <ClInclude Include="include\zencore\thread.h" /> + <ClInclude Include="include\zencore\string.h" /> + <ClInclude Include="include\zencore\streamutil.h" /> + <ClInclude Include="include\zencore\stream.h" /> + <ClInclude Include="include\zencore\stats.h" /> + <ClInclude Include="include\zencore\blake3.h" /> + <ClInclude Include="include\zencore\atomic.h" /> + <ClInclude Include="include\zencore\enumflags.h" /> + <ClInclude Include="include\zencore\except.h" /> + <ClInclude Include="include\zencore\filesystem.h" /> + <ClInclude Include="include\zencore\httpserver.h" /> + <ClInclude Include="include\zencore\refcount.h" /> + <ClInclude Include="include\zencore\memory.h" /> + <ClInclude Include="include\zencore\windows.h" /> + <ClInclude Include="include\zencore\iobuffer.h" /> + <ClInclude Include="include\zencore\sharedbuffer.h" /> + <ClInclude Include="include\zencore\iohash.h" /> + <ClInclude Include="include\zencore\compactbinarybuilder.h" /> + <ClInclude Include="include\zencore\compactbinarypackage.h" /> + <ClInclude Include="include\zencore\compactbinaryvalidation.h" /> + <ClInclude Include="include\zencore\httpclient.h" /> + <ClInclude Include="include\zencore\md5.h" /> + <ClInclude Include="include\zencore\fmtutils.h" /> + <ClInclude Include="include\zencore\xxhash.h" /> + <ClInclude Include="iothreadpool.h" /> + <ClInclude Include="include\zencore\varint.h" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="snapshot_manifest.cpp" /> + <ClCompile Include="sha1.cpp" /> + <ClCompile Include="zencore.cpp" /> + <ClCompile Include="compactbinary.cpp" /> + <ClCompile Include="uid.cpp" /> + <ClCompile Include="blake3.cpp" /> + <ClCompile Include="filesystem.cpp" /> + <ClCompile Include="httpserver.cpp" /> + <ClCompile Include="memory.cpp" /> + <ClCompile Include="refcount.cpp" /> + <ClCompile Include="stats.cpp" /> + <ClCompile Include="stream.cpp" /> + <ClCompile Include="streamutil.cpp" /> + <ClCompile Include="string.cpp" /> + <ClCompile Include="thread.cpp" /> + <ClCompile Include="timer.cpp" /> + <ClCompile Include="trace.cpp" /> + <ClCompile Include="iobuffer.cpp" /> + <ClCompile Include="sharedbuffer.cpp" /> + <ClCompile Include="iohash.cpp" /> + <ClCompile Include="compactbinarybuilder.cpp" /> + <ClCompile Include="compactbinarypackage.cpp" /> + <ClCompile Include="compactbinaryvalidation.cpp" /> + <ClCompile Include="httpclient.cpp" /> + <ClCompile Include="md5.cpp" /> + <ClCompile Include="except.cpp" /> + <ClCompile Include="xxhash.cpp" /> + <ClCompile Include="iothreadpool.cpp" /> + <ClCompile Include="compress.cpp" /> + </ItemGroup> + <ItemGroup> + <Filter Include="CAS"> + <UniqueIdentifier>{af5266fa-37a5-494c-9116-b15a3e6edd29}</UniqueIdentifier> + </Filter> + </ItemGroup> +</Project>
\ No newline at end of file |