diff options
Diffstat (limited to 'zencore')
83 files changed, 0 insertions, 25031 deletions
diff --git a/zencore/.gitignore b/zencore/.gitignore deleted file mode 100644 index 77d39c17e..000000000 --- a/zencore/.gitignore +++ /dev/null @@ -1 +0,0 @@ -include/zencore/config.h diff --git a/zencore/base64.cpp b/zencore/base64.cpp deleted file mode 100644 index b97dfebbf..000000000 --- a/zencore/base64.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/base64.h> - -namespace zen { - -/** The table used to encode a 6 bit value as an ascii character */ -static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; - -/** The table used to convert an ascii character into a 6 bit value */ -#if 0 -static const uint8_t DecodingAlphabet[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x3E, 0xFF, 0xFF, 0xFF, 0x3F, // 0x20-0x2f - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x30-0x3f - 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, // 0x40-0x4f - 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x50-0x5f - 0xFF, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, // 0x60-0x6f - 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x70-0x7f - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x80-0x8f - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x90-0x9f - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xa0-0xaf - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xb0-0xbf - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xc0-0xcf - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xd0-0xdf - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff -}; -#endif // 0 - -template<typename CharType> -uint32_t -Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest) -{ - CharType* EncodedBytes = Dest; - - // Loop through the buffer converting 3 bytes of binary data at a time - while (Length >= 3) - { - uint8_t A = *Source++; - uint8_t B = *Source++; - uint8_t C = *Source++; - Length -= 3; - - // The algorithm takes 24 bits of data (3 bytes) and breaks it into 4 6bit chunks represented as ascii - uint32_t ByteTriplet = A << 16 | B << 8 | C; - - // Use the 6bit block to find the representation ascii character for it - EncodedBytes[3] = EncodingAlphabet[ByteTriplet & 0x3F]; - ByteTriplet >>= 6; - EncodedBytes[2] = EncodingAlphabet[ByteTriplet & 0x3F]; - ByteTriplet >>= 6; - EncodedBytes[1] = EncodingAlphabet[ByteTriplet & 0x3F]; - ByteTriplet >>= 6; - EncodedBytes[0] = EncodingAlphabet[ByteTriplet & 0x3F]; - - // Now we can append this buffer to our destination string - EncodedBytes += 4; - } - - // Since this algorithm operates on blocks, we may need to pad the last chunks - if (Length > 0) - { - uint8_t A = *Source++; - uint8_t B = 0; - uint8_t C = 0; - // Grab the second character if it is a 2 uint8_t finish - if (Length == 2) - { - B = *Source; - } - uint32_t ByteTriplet = A << 16 | B << 8 | C; - // Pad with = to make a 4 uint8_t chunk - EncodedBytes[3] = '='; - ByteTriplet >>= 6; - // If there's only one 1 uint8_t left in the source, then you need 2 pad chars - if (Length == 1) - { - EncodedBytes[2] = '='; - } - else - { - EncodedBytes[2] = EncodingAlphabet[ByteTriplet & 0x3F]; - } - // Now encode the remaining bits the same way - ByteTriplet >>= 6; - EncodedBytes[1] = EncodingAlphabet[ByteTriplet & 0x3F]; - ByteTriplet >>= 6; - EncodedBytes[0] = EncodingAlphabet[ByteTriplet & 0x3F]; - - EncodedBytes += 4; - } - - // Add a null terminator - *EncodedBytes = 0; - - return uint32_t(EncodedBytes - Dest); -} - -template ZENCORE_API uint32_t Base64::Encode<char>(const uint8_t* Source, uint32_t Length, char* Dest); -template ZENCORE_API uint32_t Base64::Encode<wchar_t>(const uint8_t* Source, uint32_t Length, wchar_t* Dest); - -} // namespace zen diff --git a/zencore/blake3.cpp b/zencore/blake3.cpp deleted file mode 100644 index 89826ae5d..000000000 --- a/zencore/blake3.cpp +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/blake3.h> - -#include <zencore/compositebuffer.h> -#include <zencore/string.h> -#include <zencore/testing.h> -#include <zencore/zencore.h> - -#include <string.h> - -#include "blake3.h" - -////////////////////////////////////////////////////////////////////////// - -namespace zen { - -void -blake3_forcelink() -{ -} - -BLAKE3 BLAKE3::Zero; // Initialized to all zeroes - -BLAKE3 -BLAKE3::HashMemory(const void* data, size_t byteCount) -{ - BLAKE3 b3; - - blake3_hasher b3h; - blake3_hasher_init(&b3h); - blake3_hasher_update(&b3h, data, byteCount); - blake3_hasher_finalize(&b3h, b3.Hash, sizeof b3.Hash); - - return b3; -} - -BLAKE3 -BLAKE3::HashBuffer(const CompositeBuffer& Buffer) -{ - BLAKE3 Hash; - - blake3_hasher Hasher; - blake3_hasher_init(&Hasher); - - for (const SharedBuffer& Segment : Buffer.GetSegments()) - { - blake3_hasher_update(&Hasher, Segment.GetData(), Segment.GetSize()); - } - - blake3_hasher_finalize(&Hasher, Hash.Hash, sizeof Hash.Hash); - - return Hash; -} - -BLAKE3 -BLAKE3::FromHexString(const char* string) -{ - BLAKE3 b3; - - ParseHexBytes(string, 2 * sizeof b3.Hash, b3.Hash); - - return b3; -} - -const char* -BLAKE3::ToHexString(char* outString /* 40 characters + NUL terminator */) const -{ - ToHexBytes(Hash, sizeof(BLAKE3), outString); - outString[2 * sizeof(BLAKE3)] = '\0'; - - return outString; -} - -StringBuilderBase& -BLAKE3::ToHexString(StringBuilderBase& outBuilder) const -{ - char str[65]; - ToHexString(str); - - outBuilder.AppendRange(str, &str[65]); - - return outBuilder; -} - -BLAKE3Stream::BLAKE3Stream() -{ - blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState); - static_assert(sizeof(blake3_hasher) <= sizeof(m_HashState)); - blake3_hasher_init(b3h); -} - -void -BLAKE3Stream::Reset() -{ - blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState); - blake3_hasher_init(b3h); -} - -BLAKE3Stream& -BLAKE3Stream::Append(const void* data, size_t byteCount) -{ - blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState); - blake3_hasher_update(b3h, data, byteCount); - - return *this; -} - -BLAKE3 -BLAKE3Stream::GetHash() -{ - BLAKE3 b3; - - blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState); - blake3_hasher_finalize(b3h, b3.Hash, sizeof b3.Hash); - - return b3; -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -// doctest::String -// toString(const BLAKE3& value) -// { -// char text[2 * sizeof(BLAKE3) + 1]; -// value.ToHexString(text); - -// return text; -// } - -TEST_CASE("BLAKE3") -{ - SUBCASE("Basics") - { - BLAKE3 b3 = BLAKE3::HashMemory(nullptr, 0); - CHECK(BLAKE3::FromHexString("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262") == b3); - - BLAKE3::String_t b3s; - std::string b3ss = b3.ToHexString(b3s); - CHECK(b3ss == "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"); - } - - SUBCASE("hashes") - { - CHECK(BLAKE3::FromHexString("00307ced6a8b278d5e3a9f77b138d0e9d2209717c9d45b205f427a73565cc5fb") == BLAKE3::HashMemory("abc123", 6)); - CHECK(BLAKE3::FromHexString("a7142c8c3905cd11b1e35105c7ac588b75d6798822f71e1145187ad46f3e8df4") == - BLAKE3::HashMemory("1234567890123456789012345678901234567890", 40)); - CHECK(BLAKE3::FromHexString("70e708532559265c4662d0285e5e0a4be8bd972bd1f255a93ddf342243adc427") == - BLAKE3::HashMemory("The HttpSendHttpResponse function sends an HTTP response to the specified HTTP request.", 87)); - } - - SUBCASE("streamHashes") - { - auto streamHash = [](const void* data, size_t dataBytes) -> BLAKE3 { - BLAKE3Stream b3s; - b3s.Append(data, dataBytes); - return b3s.GetHash(); - }; - - CHECK(BLAKE3::FromHexString("00307ced6a8b278d5e3a9f77b138d0e9d2209717c9d45b205f427a73565cc5fb") == streamHash("abc123", 6)); - CHECK(BLAKE3::FromHexString("a7142c8c3905cd11b1e35105c7ac588b75d6798822f71e1145187ad46f3e8df4") == - streamHash("1234567890123456789012345678901234567890", 40)); - CHECK(BLAKE3::FromHexString("70e708532559265c4662d0285e5e0a4be8bd972bd1f255a93ddf342243adc427") == - streamHash("The HttpSendHttpResponse function sends an HTTP response to the specified HTTP request.", 87)); - } -} - -#endif - -} // namespace zen diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp deleted file mode 100644 index 0db9f02ea..000000000 --- a/zencore/compactbinary.cpp +++ /dev/null @@ -1,2299 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/compactbinary.h" - -#include <zencore/base64.h> -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryvalidation.h> -#include <zencore/compactbinaryvalue.h> -#include <zencore/compress.h> -#include <zencore/endian.h> -#include <zencore/fmtutils.h> -#include <zencore/stream.h> -#include <zencore/string.h> -#include <zencore/testing.h> -#include <zencore/uid.h> - -#include <fmt/format.h> -#include <string_view> - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#else -# include <time.h> -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include <json11.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}; - -double -GetJulianDay(uint64_t Ticks) -{ - return (double)(1721425.5 + Ticks / TimeSpan::TicksPerDay); -} - -bool -IsLeapYear(int Year) -{ - if ((Year % 4) == 0) - { - return (((Year % 100) != 0) || ((Year % 400) == 0)); - } - - return false; -} - -static constexpr uint64_t -GetPlatformToDateTimeBiasInSeconds() -{ -#if ZEN_PLATFORM_WINDOWS - const uint64_t PlatformEpochYear = 1601; -#else - const uint64_t PlatformEpochYear = 1970; -#endif - const uint64_t DateTimeEpochYear = 1; - return uint64_t(double(PlatformEpochYear - DateTimeEpochYear) * 365.2425) * 86400; -} - -uint64_t -DateTime::NowTicks() -{ - static constexpr uint64_t EpochBias = GetPlatformToDateTimeBiasInSeconds(); - -#if ZEN_PLATFORM_WINDOWS - FILETIME SysTime; - GetSystemTimePreciseAsFileTime(&SysTime); - return (EpochBias * TimeSpan::TicksPerSecond) + ((uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime); -#else - int64_t SecondsSinceUnixEpoch = time(nullptr); - return (EpochBias + SecondsSinceUnixEpoch) * TimeSpan::TicksPerSecond; -#endif -} - -DateTime -DateTime::Now() -{ - return DateTime{NowTicks()}; -} - -void -DateTime::Set(int Year, int Month, int Day, int Hour, int Minute, int Second, int MilliSecond) -{ - int TotalDays = 0; - - if ((Month > 2) && IsLeapYear(Year)) - { - ++TotalDays; - } - - --Year; // the current year is not a full year yet - --Month; // the current month is not a full month yet - - TotalDays += Year * 365; - TotalDays += Year / 4; // leap year day every four years... - TotalDays -= Year / 100; // ...except every 100 years... - TotalDays += Year / 400; // ...but also every 400 years - TotalDays += DaysToMonth[Month]; // days in this year up to last month - TotalDays += Day - 1; // days in this month minus today - - Ticks = TotalDays * TimeSpan::TicksPerDay + Hour * TimeSpan::TicksPerHour + Minute * TimeSpan::TicksPerMinute + - Second * TimeSpan::TicksPerSecond + MilliSecond * TimeSpan::TicksPerMillisecond; -} - -int -DateTime::GetYear() const -{ - int Year, Month, Day; - GetDate(Year, Month, Day); - - return Year; -} - -int -DateTime::GetMonth() const -{ - int Year, Month, Day; - GetDate(Year, Month, Day); - - return Month; -} - -int -DateTime::GetDay() const -{ - int Year, Month, Day; - GetDate(Year, Month, Day); - - return Day; -} - -int -DateTime::GetHour() const -{ - return (int)((Ticks / TimeSpan::TicksPerHour) % 24); -} - -int -DateTime::GetHour12() const -{ - int Hour = GetHour(); - - if (Hour < 1) - { - return 12; - } - - if (Hour > 12) - { - return (Hour - 12); - } - - return Hour; -} - -int -DateTime::GetMinute() const -{ - return (int)((Ticks / TimeSpan::TicksPerMinute) % 60); -} - -int -DateTime::GetSecond() const -{ - return (int)((Ticks / TimeSpan::TicksPerSecond) % 60); -} - -int -DateTime::GetMillisecond() const -{ - return (int)((Ticks / TimeSpan::TicksPerMillisecond) % 1000); -} - -void -DateTime::GetDate(int& Year, int& Month, int& Day) const -{ - // Based on FORTRAN code in: - // Fliegel, H. F. and van Flandern, T. C., - // Communications of the ACM, Vol. 11, No. 10 (October 1968). - - int i, j, k, l, n; - - l = int(GetJulianDay(Ticks) + 0.5) + 68569; - n = 4 * l / 146097; - l = l - (146097 * n + 3) / 4; - i = 4000 * (l + 1) / 1461001; - l = l - 1461 * i / 4 + 31; - j = 80 * l / 2447; - k = l - 2447 * j / 80; - l = j / 11; - j = j + 2 - 12 * l; - i = 100 * (n - 49) + i + l; - - Year = i; - Month = j; - Day = k; -} - -std::string -DateTime::ToString(const char* Format) const -{ - ExtendableStringBuilder<32> Result; - int Year, Month, Day; - - GetDate(Year, Month, Day); - - if (Format != nullptr) - { - while (*Format != '\0') - { - if ((*Format == '%') && (*(++Format) != '\0')) - { - switch (*Format) - { - // case 'a': Result.Append(IsMorning() ? TEXT("am") : TEXT("pm")); break; - // case 'A': Result.Append(IsMorning() ? TEXT("AM") : TEXT("PM")); break; - case 'd': - Result.Append(fmt::format("{:02}", Day)); - break; - // case 'D': Result.Appendf(TEXT("%03i"), GetDayOfYear()); break; - case 'm': - Result.Append(fmt::format("{:02}", Month)); - break; - case 'y': - Result.Append(fmt::format("{:02}", Year % 100)); - break; - case 'Y': - Result.Append(fmt::format("{:04}", Year)); - break; - case 'h': - Result.Append(fmt::format("{:02}", GetHour12())); - break; - case 'H': - Result.Append(fmt::format("{:02}", GetHour())); - break; - case 'M': - Result.Append(fmt::format("{:02}", GetMinute())); - break; - case 'S': - Result.Append(fmt::format("{:02}", GetSecond())); - break; - case 's': - Result.Append(fmt::format("{:03}", GetMillisecond())); - break; - default: - Result.Append(*Format); - } - } - else - { - Result.Append(*Format); - } - - // move to the next one - Format++; - } - } - - return Result.ToString(); -} - -std::string -DateTime::ToIso8601() const -{ - return ToString("%Y-%m-%dT%H:%M:%S.%sZ"); -} - -void -TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano) -{ - int64_t TotalTicks = 0; - - TotalTicks += Days * TicksPerDay; - TotalTicks += Hours * TicksPerHour; - TotalTicks += Minutes * TicksPerMinute; - TotalTicks += Seconds * TicksPerSecond; - TotalTicks += FractionNano / NanosecondsPerTick; - - Ticks = TotalTicks; -} - -std::string -TimeSpan::ToString(const char* Format) const -{ - StringBuilder<128> Result; - - Result.Append((int64_t(Ticks) < 0) ? '-' : '+'); - - while (*Format != '\0') - { - if ((*Format == '%') && (*++Format != '\0')) - { - switch (*Format) - { - case 'd': - Result.Append(fmt::format("{}", GetDays())); - break; - case 'D': - Result.Append(fmt::format("{:08}", GetDays())); - break; - case 'h': - Result.Append(fmt::format("{:02}", GetHours())); - break; - case 'm': - Result.Append(fmt::format("{:02}", GetMinutes())); - break; - case 's': - Result.Append(fmt::format("{:02}", GetSeconds())); - break; - case 'f': - Result.Append(fmt::format("{:03}", GetFractionMilli())); - break; - case 'u': - Result.Append(fmt::format("{:06}", GetFractionMicro())); - break; - case 't': - Result.Append(fmt::format("{:07}", GetFractionTicks())); - break; - case 'n': - Result.Append(fmt::format("{:09}", GetFractionNano())); - break; - default: - Result.Append(*Format); - } - } - else - { - Result.Append(*Format); - } - - ++Format; - } - - return Result.ToString(); -} - -std::string -TimeSpan::ToString() const -{ - if (GetDays() == 0) - { - return ToString("%h:%m:%s.%f"); - } - - return ToString("%d.%h:%m:%s.%f"); -} - -StringBuilderBase& -Guid::ToString(StringBuilderBase& Sb) const -{ - char Buf[128]; - snprintf(Buf, sizeof Buf, "%08x-%04x-%04x-%04x-%04x%08x", A, B >> 16, B & 0xFFFF, C >> 16, C & 0xFFFF, D); - Sb << Buf; - - return Sb; -} - -////////////////////////////////////////////////////////////////////////// - -namespace CompactBinaryPrivate { - static constexpr const uint8_t GEmptyObjectPayload[] = {uint8_t(CbFieldType::Object), 0x00}; - static constexpr const uint8_t GEmptyArrayPayload[] = {uint8_t(CbFieldType::Array), 0x01, 0x00}; -} // namespace CompactBinaryPrivate - -////////////////////////////////////////////////////////////////////////// - -CbFieldView::CbFieldView(const void* DataPointer, CbFieldType FieldType) -{ - const uint8_t* Bytes = static_cast<const uint8_t*>(DataPointer); - const CbFieldType LocalType = CbFieldTypeOps::HasFieldType(FieldType) ? (CbFieldType(*Bytes++) | CbFieldType::HasFieldType) : FieldType; - - uint32_t NameLenByteCount = 0; - const uint64_t NameLen64 = CbFieldTypeOps::HasFieldName(LocalType) ? ReadVarUInt(Bytes, NameLenByteCount) : 0; - Bytes += NameLen64 + NameLenByteCount; - - Type = LocalType; - NameLen = uint32_t(std::clamp<uint64_t>(NameLen64, 0, ~uint32_t(0))); - Payload = Bytes; -} - -void -CbFieldView::IterateAttachments(std::function<void(CbFieldView)> Visitor) const -{ - switch (CbFieldTypeOps::GetType(Type)) - { - case CbFieldType::Object: - case CbFieldType::UniformObject: - return CbObjectView::FromFieldView(*this).IterateAttachments(Visitor); - case CbFieldType::Array: - case CbFieldType::UniformArray: - return CbArrayView::FromFieldView(*this).IterateAttachments(Visitor); - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - return Visitor(*this); - default: - return; - } -} - -CbObjectView -CbFieldView::AsObjectView() -{ - if (CbFieldTypeOps::IsObject(Type)) - { - Error = CbFieldError::None; - return CbObjectView::FromFieldView(*this); - } - else - { - Error = CbFieldError::TypeError; - return CbObjectView(); - } -} - -CbArrayView -CbFieldView::AsArrayView() -{ - if (CbFieldTypeOps::IsArray(Type)) - { - Error = CbFieldError::None; - return CbArrayView::FromFieldView(*this); - } - else - { - Error = CbFieldError::TypeError; - return CbArrayView(); - } -} - -MemoryView -CbFieldView::AsBinaryView(const MemoryView Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsBinary(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsBinary(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -std::string_view -CbFieldView::AsString(const std::string_view Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsString(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsString(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -std::u8string_view -CbFieldView::AsU8String(const std::u8string_view Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsString(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsU8String(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -uint64_t -CbFieldView::AsInteger(const uint64_t Default, const CompactBinaryPrivate::IntegerParams Params) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsInteger(Accessor.GetType())) - { - return Accessor.AsInteger(Params, &Error, Default); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -float -CbFieldView::AsFloat(const float Default) -{ - switch (CbValue Accessor = GetValue(); Accessor.GetType()) - { - case CbFieldType::IntegerPositive: - case CbFieldType::IntegerNegative: - { - const uint64_t IsNegative = uint8_t(Accessor.GetType()) & 1; - constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*FLT_MANT_DIG*/ 24) - 1); - - uint32_t MagnitudeByteCount; - const int64_t Magnitude = ReadVarUInt(Accessor.GetData(), MagnitudeByteCount) + IsNegative; - const uint64_t IsInRange = !(Magnitude & OutOfRangeMask); - Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError; - return IsInRange ? float(IsNegative ? -Magnitude : Magnitude) : Default; - } - case CbFieldType::Float32: - { - Error = CbFieldError::None; - return Accessor.AsFloat32(); - } - case CbFieldType::Float64: - Error = CbFieldError::RangeError; - return Default; - default: - Error = CbFieldError::TypeError; - return Default; - } -} - -double -CbFieldView::AsDouble(const double Default) -{ - switch (CbValue Accessor = GetValue(); Accessor.GetType()) - { - case CbFieldType::IntegerPositive: - case CbFieldType::IntegerNegative: - { - const uint64_t IsNegative = uint8_t(Accessor.GetType()) & 1; - constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*DBL_MANT_DIG*/ 53) - 1); - - uint32_t MagnitudeByteCount; - const int64_t Magnitude = ReadVarUInt(Accessor.GetData(), MagnitudeByteCount) + IsNegative; - const uint64_t IsInRange = !(Magnitude & OutOfRangeMask); - Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError; - return IsInRange ? double(IsNegative ? -Magnitude : Magnitude) : Default; - } - case CbFieldType::Float32: - { - Error = CbFieldError::None; - return Accessor.AsFloat32(); - } - case CbFieldType::Float64: - { - Error = CbFieldError::None; - return Accessor.AsFloat64(); - } - default: - Error = CbFieldError::TypeError; - return Default; - } -} - -bool -CbFieldView::AsBool(const bool bDefault) -{ - CbValue Accessor = GetValue(); - const bool IsBool = CbFieldTypeOps::IsBool(Accessor.GetType()); - Error = IsBool ? CbFieldError::None : CbFieldError::TypeError; - return (uint8_t(IsBool) & Accessor.AsBool()) | ((!IsBool) & bDefault); -} - -IoHash -CbFieldView::AsObjectAttachment(const IoHash& Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsObjectAttachment(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsObjectAttachment(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -IoHash -CbFieldView::AsBinaryAttachment(const IoHash& Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsBinaryAttachment(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsBinaryAttachment(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -IoHash -CbFieldView::AsAttachment(const IoHash& Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsAttachment(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsAttachment(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -IoHash -CbFieldView::AsHash(const IoHash& Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsHash(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsHash(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -Guid -CbFieldView::AsUuid() -{ - return AsUuid(Guid()); -} - -Guid -CbFieldView::AsUuid(const Guid& Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsUuid(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsUuid(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -Oid -CbFieldView::AsObjectId() -{ - return AsObjectId(Oid()); -} - -Oid -CbFieldView::AsObjectId(const Oid& Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsObjectId(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsObjectId(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -CbCustomById -CbFieldView::AsCustomById(CbCustomById Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsCustomById(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsCustomById(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -CbCustomByName -CbFieldView::AsCustomByName(CbCustomByName Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsCustomByName(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsCustomByName(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -int64_t -CbFieldView::AsDateTimeTicks(const int64_t Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsDateTime(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsDateTimeTicks(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -DateTime -CbFieldView::AsDateTime() -{ - return DateTime(AsDateTimeTicks(0)); -} - -DateTime -CbFieldView::AsDateTime(DateTime Default) -{ - return DateTime(AsDateTimeTicks(Default.GetTicks())); -} - -int64_t -CbFieldView::AsTimeSpanTicks(const int64_t Default) -{ - if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsTimeSpan(Accessor.GetType())) - { - Error = CbFieldError::None; - return Accessor.AsTimeSpanTicks(); - } - else - { - Error = CbFieldError::TypeError; - return Default; - } -} - -TimeSpan -CbFieldView::AsTimeSpan() -{ - return TimeSpan(AsTimeSpanTicks(0)); -} - -TimeSpan -CbFieldView::AsTimeSpan(TimeSpan Default) -{ - return TimeSpan(AsTimeSpanTicks(Default.GetTicks())); -} - -uint64_t -CbFieldView::GetSize() const -{ - return sizeof(CbFieldType) + GetViewNoType().GetSize(); -} - -uint64_t -CbFieldView::GetPayloadSize() const -{ - switch (CbFieldTypeOps::GetType(Type)) - { - case CbFieldType::None: - case CbFieldType::Null: - return 0; - case CbFieldType::Object: - case CbFieldType::UniformObject: - case CbFieldType::Array: - case CbFieldType::UniformArray: - case CbFieldType::Binary: - case CbFieldType::String: - { - uint32_t PayloadSizeByteCount; - const uint64_t PayloadSize = ReadVarUInt(Payload, PayloadSizeByteCount); - return PayloadSize + PayloadSizeByteCount; - } - case CbFieldType::IntegerPositive: - case CbFieldType::IntegerNegative: - return MeasureVarUInt(Payload); - case CbFieldType::Float32: - return 4; - case CbFieldType::Float64: - return 8; - case CbFieldType::BoolFalse: - case CbFieldType::BoolTrue: - return 0; - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - case CbFieldType::Hash: - return 20; - case CbFieldType::Uuid: - return 16; - case CbFieldType::ObjectId: - return 12; - case CbFieldType::DateTime: - case CbFieldType::TimeSpan: - return 8; - default: - return 0; - } -} - -IoHash -CbFieldView::GetHash() const -{ - IoHashStream HashStream; - GetHash(HashStream); - return HashStream.GetHash(); -} - -void -CbFieldView::GetHash(IoHashStream& Hash) const -{ - const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type); - Hash.Append(&SerializedType, sizeof(SerializedType)); - auto View = GetViewNoType(); - Hash.Append(View.GetData(), View.GetSize()); -} - -bool -CbFieldView::Equals(const CbFieldView& Other) const -{ - return CbFieldTypeOps::GetSerializedType(Type) == CbFieldTypeOps::GetSerializedType(Other.Type) && - GetViewNoType().EqualBytes(Other.GetViewNoType()); -} - -void -CbFieldView::CopyTo(MutableMemoryView Buffer) const -{ - const MemoryView Source = GetViewNoType(); - ZEN_ASSERT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize()); - // TEXT("A buffer of %" UINT64_FMT " bytes was provided when %" UINT64_FMT " bytes are required"), - // Buffer.GetSize(), - // sizeof(CbFieldType) + Source.GetSize()); - *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetSerializedType(Type); - Buffer.RightChopInline(sizeof(CbFieldType)); - memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize()); -} - -void -CbFieldView::CopyTo(BinaryWriter& Ar) const -{ - const MemoryView SourceView = GetViewNoType(); - CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type); - const MemoryView TypeView(reinterpret_cast<const uint8_t*>(&SerializedType), sizeof(SerializedType)); - Ar.Write({TypeView, SourceView}); -} - -MemoryView -CbFieldView::GetView() const -{ - const uint32_t TypeSize = CbFieldTypeOps::HasFieldType(Type) ? sizeof(CbFieldType) : 0; - const uint32_t NameSize = CbFieldTypeOps::HasFieldName(Type) ? NameLen + MeasureVarUInt(NameLen) : 0; - const uint64_t PayloadSize = GetPayloadSize(); - return MemoryView(static_cast<const uint8_t*>(Payload) - TypeSize - NameSize, TypeSize + NameSize + PayloadSize); -} - -MemoryView -CbFieldView::GetViewNoType() const -{ - const uint32_t NameSize = CbFieldTypeOps::HasFieldName(Type) ? NameLen + MeasureVarUInt(NameLen) : 0; - const uint64_t PayloadSize = GetPayloadSize(); - return MemoryView(static_cast<const uint8_t*>(Payload) - NameSize, NameSize + PayloadSize); -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -CbArrayView::CbArrayView() : CbFieldView(CompactBinaryPrivate::GEmptyArrayPayload) -{ -} - -uint64_t -CbArrayView::Num() const -{ - const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload()); - PayloadBytes += MeasureVarUInt(PayloadBytes); - uint32_t NumByteCount; - return ReadVarUInt(PayloadBytes, NumByteCount); -} - -CbFieldViewIterator -CbArrayView::CreateViewIterator() const -{ - const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload()); - uint32_t PayloadSizeByteCount; - const uint64_t PayloadSize = ReadVarUInt(PayloadBytes, PayloadSizeByteCount); - PayloadBytes += PayloadSizeByteCount; - const uint64_t NumByteCount = MeasureVarUInt(PayloadBytes); - if (PayloadSize > NumByteCount) - { - const void* const PayloadEnd = PayloadBytes + PayloadSize; - PayloadBytes += NumByteCount; - const CbFieldType UniformType = - CbFieldTypeOps::GetType(GetType()) == CbFieldType::UniformArray ? CbFieldType(*PayloadBytes++) : CbFieldType::HasFieldType; - return CbFieldViewIterator::MakeRange(MemoryView(PayloadBytes, PayloadEnd), UniformType); - } - return CbFieldViewIterator(); -} - -void -CbArrayView::VisitFields(ICbVisitor&) -{ -} - -uint64_t -CbArrayView::GetSize() const -{ - return sizeof(CbFieldType) + GetPayloadSize(); -} - -IoHash -CbArrayView::GetHash() const -{ - IoHashStream Hash; - GetHash(Hash); - return Hash.GetHash(); -} - -void -CbArrayView::GetHash(IoHashStream& HashStream) const -{ - const CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType()); - HashStream.Append(&SerializedType, sizeof(SerializedType)); - auto _ = GetPayloadView(); - HashStream.Append(_.GetData(), _.GetSize()); -} - -bool -CbArrayView::Equals(const CbArrayView& Other) const -{ - return CbFieldTypeOps::GetType(GetType()) == CbFieldTypeOps::GetType(Other.GetType()) && - GetPayloadView().EqualBytes(Other.GetPayloadView()); -} - -void -CbArrayView::CopyTo(MutableMemoryView Buffer) const -{ - const MemoryView Source = GetPayloadView(); - ZEN_ASSERT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize()); - // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."), - // Buffer.GetSize(), - // sizeof(CbFieldType) + Source.GetSize()); - - *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetType(GetType()); - Buffer.RightChopInline(sizeof(CbFieldType)); - memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize()); -} - -void -CbArrayView::CopyTo(BinaryWriter& Ar) const -{ - const MemoryView SourceView = GetPayloadView(); - CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(GetType()); - const MemoryView TypeView(reinterpret_cast<const uint8_t*>(&SerializedType), sizeof(SerializedType)); - Ar.Write({TypeView, SourceView}); -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -CbObjectView::CbObjectView() : CbFieldView(CompactBinaryPrivate::GEmptyObjectPayload) -{ -} - -CbFieldViewIterator -CbObjectView::CreateViewIterator() const -{ - const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload()); - uint32_t PayloadSizeByteCount; - const uint64_t PayloadSize = ReadVarUInt(PayloadBytes, PayloadSizeByteCount); - - PayloadBytes += PayloadSizeByteCount; - - if (PayloadSize) - { - const void* const PayloadEnd = PayloadBytes + PayloadSize; - const CbFieldType UniformType = - CbFieldTypeOps::GetType(GetType()) == CbFieldType::UniformObject ? CbFieldType(*PayloadBytes++) : CbFieldType::HasFieldType; - return CbFieldViewIterator::MakeRange(MemoryView(PayloadBytes, PayloadEnd), UniformType); - } - - return CbFieldViewIterator(); -} - -void -CbObjectView::VisitFields(ICbVisitor&) -{ -} - -CbFieldView -CbObjectView::FindView(const std::string_view Name) const -{ - for (const CbFieldView& Field : *this) - { - if (Name == Field.GetName()) - { - return Field; - } - } - return CbFieldView(); -} - -CbFieldView -CbObjectView::FindViewIgnoreCase(const std::string_view Name) const -{ - for (const CbFieldView& Field : *this) - { - if (Name == Field.GetName()) - { - return Field; - } - } - return CbFieldView(); -} - -CbObjectView::operator bool() const -{ - return GetSize() > sizeof(CompactBinaryPrivate::GEmptyObjectPayload); -} - -uint64_t -CbObjectView::GetSize() const -{ - return sizeof(CbFieldType) + GetPayloadSize(); -} - -IoHash -CbObjectView::GetHash() const -{ - IoHashStream Hash; - GetHash(Hash); - return Hash.GetHash(); -} - -void -CbObjectView::GetHash(IoHashStream& HashStream) const -{ - const CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType()); - HashStream.Append(&SerializedType, sizeof(SerializedType)); - HashStream.Append(GetPayloadView()); -} - -bool -CbObjectView::Equals(const CbObjectView& Other) const -{ - return CbFieldTypeOps::GetType(GetType()) == CbFieldTypeOps::GetType(Other.GetType()) && - GetPayloadView().EqualBytes(Other.GetPayloadView()); -} - -void -CbObjectView::CopyTo(MutableMemoryView Buffer) const -{ - const MemoryView Source = GetPayloadView(); - ZEN_ASSERT(Buffer.GetSize() == (sizeof(CbFieldType) + Source.GetSize())); - // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."), - // Buffer.GetSize(), - // sizeof(CbFieldType) + Source.GetSize()); - *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetType(GetType()); - Buffer.RightChopInline(sizeof(CbFieldType)); - memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize()); -} - -void -CbObjectView::CopyTo(BinaryWriter& Ar) const -{ - const MemoryView SourceView = GetPayloadView(); - CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(GetType()); - const MemoryView TypeView(reinterpret_cast<const uint8_t*>(&SerializedType), sizeof(SerializedType)); - Ar.Write({TypeView, SourceView}); -} - -////////////////////////////////////////////////////////////////////////// - -template<typename FieldType> -uint64_t -TCbFieldIterator<FieldType>::GetRangeSize() const -{ - MemoryView View; - if (TryGetSerializedRangeView(View)) - { - return View.GetSize(); - } - else - { - uint64_t Size = 0; - for (CbFieldViewIterator It(*this); It; ++It) - { - Size += It.GetSize(); - } - return Size; - } -} - -template<typename FieldType> -IoHash -TCbFieldIterator<FieldType>::GetRangeHash() const -{ - IoHashStream Hash; - GetRangeHash(Hash); - return IoHash(Hash.GetHash()); -} - -template<typename FieldType> -void -TCbFieldIterator<FieldType>::GetRangeHash(IoHashStream& Hash) const -{ - MemoryView View; - if (TryGetSerializedRangeView(View)) - { - Hash.Append(View.GetData(), View.GetSize()); - } - else - { - for (CbFieldViewIterator It(*this); It; ++It) - { - It.GetHash(Hash); - } - } -} - -template<typename FieldType> -void -TCbFieldIterator<FieldType>::CopyRangeTo(MutableMemoryView InBuffer) const -{ - MemoryView Source; - if (TryGetSerializedRangeView(Source)) - { - ZEN_ASSERT(InBuffer.GetSize() == Source.GetSize()); - // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."), - // InBuffer.GetSize(), - // Source.GetSize()); - memcpy(InBuffer.GetData(), Source.GetData(), Source.GetSize()); - } - else - { - for (CbFieldViewIterator It(*this); It; ++It) - { - const uint64_t Size = It.GetSize(); - It.CopyTo(InBuffer.Left(Size)); - InBuffer.RightChopInline(Size); - } - } -} - -template class TCbFieldIterator<CbFieldView>; -template class TCbFieldIterator<CbField>; - -template<typename FieldType> -void -TCbFieldIterator<FieldType>::IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const -{ - if (CbFieldTypeOps::HasFieldType(FieldType::GetType())) - { - // Always iterate over non-uniform ranges because we do not know if they contain an attachment. - for (CbFieldViewIterator It(*this); It; ++It) - { - if (CbFieldTypeOps::MayContainAttachments(It.GetType())) - { - It.IterateAttachments(Visitor); - } - } - } - else - { - // Only iterate over uniform ranges if the uniform type may contain an attachment. - if (CbFieldTypeOps::MayContainAttachments(FieldType::GetType())) - { - for (CbFieldViewIterator It(*this); It; ++It) - { - It.IterateAttachments(Visitor); - } - } - } -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -CbFieldIterator -CbFieldIterator::CloneRange(const CbFieldViewIterator& It) -{ - MemoryView View; - if (It.TryGetSerializedRangeView(View)) - { - return MakeRange(SharedBuffer::Clone(View)); - } - else - { - UniqueBuffer Buffer = UniqueBuffer::Alloc(It.GetRangeSize()); - It.CopyRangeTo(MutableMemoryView(Buffer.GetData(), Buffer.GetSize())); - return MakeRange(SharedBuffer(std::move(Buffer))); - } -} - -SharedBuffer -CbFieldIterator::GetRangeBuffer() const -{ - const MemoryView RangeView = GetRangeView(); - const SharedBuffer& OuterBuffer = GetOuterBuffer(); - return OuterBuffer.GetView() == RangeView ? OuterBuffer : SharedBuffer::MakeView(RangeView, OuterBuffer); -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -uint64_t -MeasureCompactBinary(MemoryView View, CbFieldType Type) -{ - uint64_t Size; - return TryMeasureCompactBinary(View, Type, Size, Type) ? Size : 0; -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -bool -TryMeasureCompactBinary(MemoryView View, CbFieldType& OutType, uint64_t& OutSize, CbFieldType Type) -{ - uint64_t Size = 0; - - if (CbFieldTypeOps::HasFieldType(Type)) - { - if (View.GetSize() == 0) - { - OutType = CbFieldType::None; - OutSize = 1; - return false; - } - - Type = *static_cast<const CbFieldType*>(View.GetData()); - View.RightChopInline(1); - Size += 1; - } - - bool bDynamicSize = false; - uint64_t FixedSize = 0; - switch (CbFieldTypeOps::GetType(Type)) - { - case CbFieldType::Null: - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - case CbFieldType::Array: - case CbFieldType::UniformArray: - case CbFieldType::Binary: - case CbFieldType::String: - case CbFieldType::IntegerPositive: - case CbFieldType::IntegerNegative: - bDynamicSize = true; - break; - case CbFieldType::Float32: - FixedSize = 4; - break; - case CbFieldType::Float64: - FixedSize = 8; - break; - case CbFieldType::BoolFalse: - case CbFieldType::BoolTrue: - break; - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - case CbFieldType::Hash: - FixedSize = 20; - break; - case CbFieldType::Uuid: - FixedSize = 16; - break; - case CbFieldType::ObjectId: - FixedSize = 12; - break; - case CbFieldType::DateTime: - case CbFieldType::TimeSpan: - FixedSize = 8; - break; - case CbFieldType::None: - default: - OutType = CbFieldType::None; - OutSize = 0; - return false; - } - - OutType = Type; - - if (CbFieldTypeOps::HasFieldName(Type)) - { - if (View.GetSize() == 0) - { - OutSize = Size + 1; - return false; - } - - uint32_t NameLenByteCount = MeasureVarUInt(View.GetData()); - if (View.GetSize() < NameLenByteCount) - { - OutSize = Size + NameLenByteCount; - return false; - } - - const uint64_t NameLen = ReadVarUInt(View.GetData(), NameLenByteCount); - const uint64_t NameSize = NameLen + NameLenByteCount; - - if (bDynamicSize && View.GetSize() < NameSize) - { - OutSize = Size + NameSize; - return false; - } - - View.RightChopInline(NameSize); - Size += NameSize; - } - - switch (CbFieldTypeOps::GetType(Type)) - { - case CbFieldType::Object: - case CbFieldType::UniformObject: - case CbFieldType::Array: - case CbFieldType::UniformArray: - case CbFieldType::Binary: - case CbFieldType::String: - if (View.GetSize() == 0) - { - OutSize = Size + 1; - return false; - } - else - { - uint32_t PayloadSizeByteCount = MeasureVarUInt(View.GetData()); - if (View.GetSize() < PayloadSizeByteCount) - { - OutSize = Size + PayloadSizeByteCount; - return false; - } - const uint64_t PayloadSize = ReadVarUInt(View.GetData(), PayloadSizeByteCount); - OutSize = Size + PayloadSize + PayloadSizeByteCount; - } - return true; - - case CbFieldType::IntegerPositive: - case CbFieldType::IntegerNegative: - if (View.GetSize() == 0) - { - OutSize = Size + 1; - return false; - } - OutSize = Size + MeasureVarUInt(View.GetData()); - return true; - - default: - OutSize = Size + FixedSize; - return true; - } -} - -////////////////////////////////////////////////////////////////////////// - -CbField -LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) -{ - std::vector<uint8_t> HeaderBytes; - CbFieldType FieldType; - uint64_t FieldSize = 1; - - for (const int64_t StartPos = Ar.CurrentOffset(); FieldSize > 0;) - { - // Read in small increments until the total field size is known, to avoid reading too far. - const int32_t ReadSize = int32_t(FieldSize - HeaderBytes.size()); - if (Ar.CurrentOffset() + ReadSize > Ar.GetSize()) - { - break; - } - - const size_t ReadOffset = HeaderBytes.size(); - HeaderBytes.resize(ReadOffset + ReadSize); - - Ar.Read(HeaderBytes.data() + ReadOffset, ReadSize); - if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes), FieldType, FieldSize)) - { - if (FieldSize <= uint64_t(Ar.Size() - StartPos)) - { - UniqueBuffer Buffer = Allocator(FieldSize); - ZEN_ASSERT(Buffer.GetSize() == FieldSize); - MutableMemoryView View = Buffer.GetMutableView(); - memcpy(View.GetData(), HeaderBytes.data(), HeaderBytes.size()); - View.RightChopInline(HeaderBytes.size()); - if (!View.IsEmpty()) - { - // Read the remainder of the field. - Ar.Read(View.GetData(), View.GetSize()); - } - if (ValidateCompactBinary(Buffer, CbValidateMode::Default) == CbValidateError::None) - { - return CbField(SharedBuffer(std::move(Buffer))); - } - } - break; - } - } - return CbField(); -} - -CbObject -LoadCompactBinaryObject(IoBuffer&& Payload) -{ - return CbObject{SharedBuffer(std::move(Payload))}; -} - -CbObject -LoadCompactBinaryObject(const IoBuffer& Payload) -{ - return CbObject{SharedBuffer(Payload)}; -} - -CbObject -LoadCompactBinaryObject(CompressedBuffer&& Payload) -{ - return CbObject{SharedBuffer(Payload.DecompressToComposite().Flatten())}; -} - -CbObject -LoadCompactBinaryObject(const CompressedBuffer& Payload) -{ - return CbObject{SharedBuffer(Payload.DecompressToComposite().Flatten())}; -} - -////////////////////////////////////////////////////////////////////////// - -void -SaveCompactBinary(BinaryWriter& Ar, const CbFieldView& Field) -{ - Field.CopyTo(Ar); -} - -void -SaveCompactBinary(BinaryWriter& Ar, const CbArrayView& Array) -{ - Array.CopyTo(Ar); -} - -void -SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object) -{ - Object.CopyTo(Ar); -} - -////////////////////////////////////////////////////////////////////////// - -class CbJsonWriter -{ -public: - explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } - - void WriteField(CbFieldView Field) - { - using namespace std::literals; - - WriteOptionalComma(); - WriteOptionalNewLine(); - - if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) - { - AppendQuotedString(Name); - Builder << ": "sv; - } - - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) - { - case CbFieldType::Null: - Builder << "null"sv; - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - { - Builder << '{'; - NewLineAndIndent << '\t'; - NeedsNewLine = true; - for (CbFieldView It : Field) - { - WriteField(It); - } - NewLineAndIndent.RemoveSuffix(1); - if (NeedsComma) - { - WriteOptionalNewLine(); - } - Builder << '}'; - } - break; - case CbFieldType::Array: - case CbFieldType::UniformArray: - { - Builder << '['; - NewLineAndIndent << '\t'; - NeedsNewLine = true; - for (CbFieldView It : Field) - { - WriteField(It); - } - NewLineAndIndent.RemoveSuffix(1); - if (NeedsComma) - { - WriteOptionalNewLine(); - } - Builder << ']'; - } - break; - case CbFieldType::Binary: - AppendBase64String(Accessor.AsBinary()); - break; - case CbFieldType::String: - AppendQuotedString(Accessor.AsU8String()); - break; - case CbFieldType::IntegerPositive: - Builder << Accessor.AsIntegerPositive(); - break; - case CbFieldType::IntegerNegative: - Builder << Accessor.AsIntegerNegative(); - break; - case CbFieldType::Float32: - { - const float Value = Accessor.AsFloat32(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.9g}", Value)); - } - else - { - Builder << "null"sv; - } - } - break; - case CbFieldType::Float64: - { - const double Value = Accessor.AsFloat64(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.17g}", Value)); - } - else - { - Builder << "null"sv; - } - } - break; - case CbFieldType::BoolFalse: - Builder << "false"sv; - break; - case CbFieldType::BoolTrue: - Builder << "true"sv; - break; - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - { - Builder << '"'; - Accessor.AsAttachment().ToHexString(Builder); - Builder << '"'; - } - break; - case CbFieldType::Hash: - { - Builder << '"'; - Accessor.AsHash().ToHexString(Builder); - Builder << '"'; - } - break; - case CbFieldType::Uuid: - { - Builder << '"'; - Accessor.AsUuid().ToString(Builder); - Builder << '"'; - } - break; - case CbFieldType::DateTime: - Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; - break; - case CbFieldType::TimeSpan: - { - const TimeSpan Span(Accessor.AsTimeSpanTicks()); - if (Span.GetDays() == 0) - { - Builder << '"' << Span.ToString("%h:%m:%s.%n") << '"'; - } - else - { - Builder << '"' << Span.ToString("%d.%h:%m:%s.%n") << '"'; - } - break; - } - case CbFieldType::ObjectId: - Builder << '"'; - Accessor.AsObjectId().ToString(Builder); - Builder << '"'; - break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); - Builder << "{ \"Id\": "; - Builder << Custom.Id; - Builder << ", \"Data\": "; - AppendBase64String(Custom.Data); - Builder << " }"; - break; - } - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); - Builder << "{ \"Name\": "; - AppendQuotedString(Custom.Name); - Builder << ", \"Data\": "; - AppendBase64String(Custom.Data); - Builder << " }"; - break; - } - default: - ZEN_ASSERT(false); - break; - } - - NeedsComma = true; - NeedsNewLine = true; - } - -private: - void WriteOptionalComma() - { - if (NeedsComma) - { - NeedsComma = false; - Builder << ','; - } - } - - void WriteOptionalNewLine() - { - if (NeedsNewLine) - { - NeedsNewLine = false; - Builder << NewLineAndIndent; - } - } - - void AppendQuotedString(std::u8string_view Value) - { - using namespace std::literals; - - const AsciiSet EscapeSet( - "\\\"\b\f\n\r\t" - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"); - - Builder << '\"'; - while (!Value.empty()) - { - std::u8string_view Verbatim = AsciiSet::FindPrefixWithout(Value, EscapeSet); - Builder << Verbatim; - - Value = Value.substr(Verbatim.size()); - - std::u8string_view Escape = AsciiSet::FindPrefixWith(Value, EscapeSet); - for (char Char : Escape) - { - switch (Char) - { - case '\\': - Builder << "\\\\"sv; - break; - case '\"': - Builder << "\\\""sv; - break; - case '\b': - Builder << "\\b"sv; - break; - case '\f': - Builder << "\\f"sv; - break; - case '\n': - Builder << "\\n"sv; - break; - case '\r': - Builder << "\\r"sv; - break; - case '\t': - Builder << "\\t"sv; - break; - default: - Builder << Char; - break; - } - } - Value = Value.substr(Escape.size()); - } - Builder << '\"'; - } - - void AppendBase64String(MemoryView Value) - { - Builder << '"'; - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - } - -private: - StringBuilderBase& Builder; - ExtendableStringBuilder<32> NewLineAndIndent; - bool NeedsComma{false}; - bool NeedsNewLine{false}; -}; - -void -CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) -{ - CbJsonWriter Writer(Builder); - Writer.WriteField(Object.AsFieldView()); -} - -void -CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) -{ - CbJsonWriter Writer(Builder); - Writer.WriteField(Array.AsFieldView()); -} - -////////////////////////////////////////////////////////////////////////// - -class CbJsonReader -{ -public: - static CbFieldIterator Read(std::string_view JsonText, std::string& Error) - { - using namespace json11; - - const Json Json = Json::parse(std::string(JsonText), Error); - - if (Error.empty()) - { - CbWriter Writer; - if (ReadField(Writer, Json, std::string_view(), Error)) - { - return Writer.Save(); - } - } - - return CbFieldIterator(); - } - -private: - static bool ReadField(CbWriter& Writer, const json11::Json& Json, const std::string_view FieldName, std::string& Error) - { - using namespace json11; - - switch (Json.type()) - { - case Json::Type::OBJECT: - { - if (FieldName.empty()) - { - Writer.BeginObject(); - } - else - { - Writer.BeginObject(FieldName); - } - - for (const auto& Kv : Json.object_items()) - { - const std::string& Name = Kv.first; - const json11::Json& Item = Kv.second; - - if (ReadField(Writer, Item, Name, Error) == false) - { - return false; - } - } - - Writer.EndObject(); - } - break; - case Json::Type::ARRAY: - { - if (FieldName.empty()) - { - Writer.BeginArray(); - } - else - { - Writer.BeginArray(FieldName); - } - - for (const json11::Json& Item : Json.array_items()) - { - if (ReadField(Writer, Item, std::string_view(), Error) == false) - { - return false; - } - } - - Writer.EndArray(); - } - break; - case Json::Type::NUL: - { - if (FieldName.empty()) - { - Writer.AddNull(); - } - else - { - Writer.AddNull(FieldName); - } - } - break; - case Json::Type::BOOL: - { - if (FieldName.empty()) - { - Writer.AddBool(Json.bool_value()); - } - else - { - Writer.AddBool(FieldName, Json.bool_value()); - } - } - break; - case Json::Type::NUMBER: - { - if (FieldName.empty()) - { - Writer.AddFloat(Json.number_value()); - } - else - { - Writer.AddFloat(FieldName, Json.number_value()); - } - } - break; - case Json::Type::STRING: - { - Oid Id; - if (TryParseObjectId(Json.string_value(), Id)) - { - if (FieldName.empty()) - { - Writer.AddObjectId(Id); - } - else - { - Writer.AddObjectId(FieldName, Id); - } - - return true; - } - - IoHash Hash; - if (TryParseIoHash(Json.string_value(), Hash)) - { - if (FieldName.empty()) - { - Writer.AddHash(Hash); - } - else - { - Writer.AddHash(FieldName, Hash); - } - - return true; - } - - if (FieldName.empty()) - { - Writer.AddString(Json.string_value()); - } - else - { - Writer.AddString(FieldName, Json.string_value()); - } - } - break; - default: - break; - } - - return true; - } - - static constexpr AsciiSet HexCharSet = AsciiSet("0123456789abcdefABCDEF"); - - static bool TryParseObjectId(std::string_view Str, Oid& Id) - { - using namespace std::literals; - - if (Str.size() == Oid::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) - { - Id = Oid::FromHexString(Str); - return true; - } - - if (Str.starts_with("0x"sv)) - { - return TryParseObjectId(Str.substr(2), Id); - } - - return false; - } - - static bool TryParseIoHash(std::string_view Str, IoHash& Hash) - { - using namespace std::literals; - - if (Str.size() == IoHash::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) - { - Hash = IoHash::FromHexString(Str); - return true; - } - - if (Str.starts_with("0x"sv)) - { - return TryParseIoHash(Str.substr(2), Hash); - } - - return false; - } -}; - -CbFieldIterator -LoadCompactBinaryFromJson(std::string_view Json, std::string& Error) -{ - if (Json.empty() == false) - { - return CbJsonReader::Read(Json, Error); - } - - return CbFieldIterator(); -} - -CbFieldIterator -LoadCompactBinaryFromJson(std::string_view Json) -{ - std::string Error; - return LoadCompactBinaryFromJson(Json, Error); -} - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_WITH_TESTS -void -uson_forcelink() -{ -} - -TEST_CASE("uson") -{ - using namespace std::literals; - - SUBCASE("CbField") - { - constexpr CbFieldView DefaultField; - static_assert(!DefaultField.HasName(), "Error in HasName()"); - static_assert(!DefaultField.HasValue(), "Error in HasValue()"); - static_assert(!DefaultField.HasError(), "Error in HasError()"); - static_assert(DefaultField.GetError() == CbFieldError::None, "Error in GetError()"); - - CHECK(DefaultField.GetSize() == 1); - CHECK(DefaultField.GetName().size() == 0); - CHECK(DefaultField.HasName() == false); - CHECK(DefaultField.HasValue() == false); - CHECK(DefaultField.HasError() == false); - CHECK(DefaultField.GetError() == CbFieldError::None); - - const uint8_t Type = (uint8_t)CbFieldType::None; - CHECK(DefaultField.GetHash() == IoHash::HashBuffer(&Type, sizeof Type)); - - CHECK(DefaultField.GetView() == MemoryView{}); - MemoryView SerializedView; - CHECK(DefaultField.TryGetSerializedView(SerializedView) == false); - } - - SUBCASE("CbField(None)") - { - CbFieldView NoneField(nullptr, CbFieldType::None); - CHECK(NoneField.GetSize() == 1); - CHECK(NoneField.GetName().size() == 0); - CHECK(NoneField.HasName() == false); - CHECK(NoneField.HasValue() == false); - CHECK(NoneField.HasError() == false); - CHECK(NoneField.GetError() == CbFieldError::None); - CHECK(NoneField.GetHash() == CbFieldView().GetHash()); - CHECK(NoneField.GetView() == MemoryView()); - MemoryView SerializedView; - CHECK(NoneField.TryGetSerializedView(SerializedView) == false); - } - - SUBCASE("CbField(None|Type|Name)") - { - constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; - CbFieldView NoneField(NoneBytes); - - CHECK(NoneField.GetSize() == sizeof(NoneBytes)); - CHECK(NoneField.GetName().compare("Name"sv) == 0); - CHECK(NoneField.HasName() == true); - CHECK(NoneField.HasValue() == false); - CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes)); - CHECK(NoneField.GetView() == MemoryView(NoneBytes, sizeof NoneBytes)); - MemoryView SerializedView; - CHECK(NoneField.TryGetSerializedView(SerializedView) == true); - CHECK(SerializedView == MemoryView(NoneBytes, sizeof NoneBytes)); - - uint8_t CopyBytes[sizeof(NoneBytes)]; - NoneField.CopyTo(MutableMemoryView(CopyBytes, sizeof CopyBytes)); - CHECK(MemoryView(NoneBytes, sizeof NoneBytes).EqualBytes(MemoryView(CopyBytes, sizeof CopyBytes))); - } - - SUBCASE("CbField(None|Type)") - { - constexpr CbFieldType FieldType = CbFieldType::None; - const char NoneBytes[] = {char(FieldType)}; - CbFieldView NoneField(NoneBytes); - - CHECK(NoneField.GetSize() == sizeof NoneBytes); - CHECK(NoneField.GetName().size() == 0); - CHECK(NoneField.HasName() == false); - CHECK(NoneField.HasValue() == false); - CHECK(NoneField.GetHash() == CbFieldView().GetHash()); - CHECK(NoneField.GetView() == MemoryView(NoneBytes, sizeof NoneBytes)); - MemoryView SerializedView; - CHECK(NoneField.TryGetSerializedView(SerializedView) == true); - CHECK(SerializedView == MemoryView(NoneBytes, sizeof NoneBytes)); - } - - SUBCASE("CbField(None|Name)") - { - constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; - CbFieldView NoneField(NoneBytes + 1, FieldType); - CHECK(NoneField.GetSize() == uint64_t(sizeof NoneBytes)); - CHECK(NoneField.GetName().compare("Name") == 0); - CHECK(NoneField.HasName() == true); - CHECK(NoneField.HasValue() == false); - CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes)); - CHECK(NoneField.GetView() == MemoryView(NoneBytes + 1, sizeof NoneBytes - 1)); - MemoryView SerializedView; - CHECK(NoneField.TryGetSerializedView(SerializedView) == false); - - uint8_t CopyBytes[sizeof(NoneBytes)]; - NoneField.CopyTo(MutableMemoryView(CopyBytes, sizeof CopyBytes)); - CHECK(MemoryView(NoneBytes, sizeof NoneBytes).EqualBytes(MemoryView(CopyBytes, sizeof CopyBytes))); - } - - SUBCASE("CbField(None|EmptyName)") - { - constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - const uint8_t NoneBytes[] = {uint8_t(FieldType), 0}; - CbFieldView NoneField(NoneBytes + 1, FieldType); - CHECK(NoneField.GetSize() == sizeof NoneBytes); - CHECK(NoneField.GetName().empty() == true); - CHECK(NoneField.HasName() == true); - CHECK(NoneField.HasValue() == false); - CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes)); - CHECK(NoneField.GetView() == MemoryView(NoneBytes + 1, sizeof NoneBytes - 1)); - MemoryView SerializedView; - CHECK(NoneField.TryGetSerializedView(SerializedView) == false); - } - - static_assert(!std::is_constructible<CbFieldView, const CbObjectView&>::value, "Invalid constructor for CbField"); - static_assert(!std::is_assignable<CbFieldView, const CbObjectView&>::value, "Invalid assignment for CbField"); - static_assert(!std::is_convertible<CbFieldView, CbObjectView>::value, "Invalid conversion to CbObject"); - static_assert(!std::is_assignable<CbObjectView, const CbFieldView&>::value, "Invalid assignment for CbObject"); - - static_assert(std::is_constructible<CbField>::value, "Missing constructor for CbField"); - static_assert(std::is_constructible<CbField, const CbField&>::value, "Missing constructor for CbField"); - static_assert(std::is_constructible<CbField, CbField&&>::value, "Missing constructor for CbField"); -} - -TEST_CASE("uson.null") -{ - using namespace std::literals; - - SUBCASE("CbField(Null)") - { - CbFieldView NullField(nullptr, CbFieldType::Null); - CHECK(NullField.GetSize() == 1); - CHECK(NullField.IsNull() == true); - CHECK(NullField.HasValue() == true); - CHECK(NullField.HasError() == false); - CHECK(NullField.GetError() == CbFieldError::None); - const uint8_t Null[]{uint8_t(CbFieldType::Null)}; - CHECK(NullField.GetHash() == IoHash::HashBuffer(Null, sizeof Null)); - } - - SUBCASE("CbField(None)") - { - CbFieldView Field; - CHECK(Field.IsNull() == false); - } -} - -TEST_CASE("uson.json") -{ - SUBCASE("string") - { - CbObjectWriter Writer; - Writer << "KeyOne" - << "ValueOne"; - Writer << "KeyTwo" - << "ValueTwo"; - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const std::string ValueOne = Json["KeyOne"].string_value(); - const std::string ValueTwo = Json["KeyTwo"].string_value(); - - CHECK(JsonError.empty()); - CHECK(ValueOne == "ValueOne"); - CHECK(ValueTwo == "ValueTwo"); - } - - SUBCASE("number") - { - const float ExpectedFloatValue = 21.21f; - const double ExpectedDoubleValue = 42.42; - - CbObjectWriter Writer; - Writer << "Float" << ExpectedFloatValue; - Writer << "Double" << ExpectedDoubleValue; - - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const float FloatValue = float(Json["Float"].number_value()); - const double DoubleValue = Json["Double"].number_value(); - - CHECK(JsonError.empty()); - CHECK(FloatValue == Approx(ExpectedFloatValue)); - CHECK(DoubleValue == Approx(ExpectedDoubleValue)); - } - - SUBCASE("number.nan") - { - const float FloatNan = std::numeric_limits<float>::quiet_NaN(); - const double DoubleNan = std::numeric_limits<double>::quiet_NaN(); - - CbObjectWriter Writer; - Writer << "FloatNan" << FloatNan; - Writer << "DoubleNan" << DoubleNan; - - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const double FloatValue = Json["FloatNan"].number_value(); - const double DoubleValue = Json["DoubleNan"].number_value(); - - CHECK(JsonError.empty()); - CHECK(FloatValue == 0); - CHECK(DoubleValue == 0); - } -} - -TEST_CASE("uson.datetime") -{ - using namespace std::literals; - - { - DateTime D1600(1601, 1, 1); - CHECK_EQ(D1600.GetYear(), 1601); - CHECK_EQ(D1600.GetMonth(), 1); - CHECK_EQ(D1600.GetDay(), 1); - CHECK_EQ(D1600.GetHour(), 0); - CHECK_EQ(D1600.GetMinute(), 0); - CHECK_EQ(D1600.GetSecond(), 0); - - CHECK_EQ(D1600.ToIso8601(), "1601-01-01T00:00:00.000Z"sv); - } - - { - DateTime D72(1972, 2, 23, 17, 30, 10); - CHECK_EQ(D72.GetYear(), 1972); - CHECK_EQ(D72.GetMonth(), 2); - CHECK_EQ(D72.GetDay(), 23); - CHECK_EQ(D72.GetHour(), 17); - CHECK_EQ(D72.GetMinute(), 30); - CHECK_EQ(D72.GetSecond(), 10); - } -} - -TEST_CASE("json.uson") -{ - using namespace std::literals; - using namespace json11; - - SUBCASE("empty") - { - CbFieldIterator It = LoadCompactBinaryFromJson(""sv); - CHECK(It.HasValue() == false); - } - - SUBCASE("object") - { - const Json JsonObject = Json::object{{"Null", nullptr}, - {"String", "Value1"}, - {"Bool", true}, - {"Number", 46.2}, - {"Array", Json::array{1, 2, 3}}, - {"Object", - Json::object{ - {"String", "Value2"}, - }}}; - - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["Null"].IsNull()); - CHECK(Cb["String"].AsString() == "Value1"sv); - CHECK(Cb["Bool"].AsBool()); - CHECK(Cb["Number"].AsDouble() == 46.2); - CHECK(Cb["Object"].IsObject()); - CbObjectView Object = Cb["Object"].AsObjectView(); - CHECK(Object["String"].AsString() == "Value2"sv); - } - - SUBCASE("array") - { - const Json JsonArray = Json::array{42, 43, 44}; - CbArray Cb = LoadCompactBinaryFromJson(JsonArray.dump()).AsArray(); - - auto It = Cb.CreateIterator(); - CHECK((*It).AsDouble() == 42); - It++; - CHECK((*It).AsDouble() == 43); - It++; - CHECK((*It).AsDouble() == 44); - } - - SUBCASE("objectid") - { - const Oid& Id = Oid::NewOid(); - - StringBuilder<64> Sb; - Id.ToString(Sb); - - Json JsonObject = Json::object{{"value", Sb.ToString()}}; - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsObjectId()); - CHECK(Cb["value"sv].AsObjectId() == Id); - - Sb.Reset(); - Sb << "0x"; - Id.ToString(Sb); - - JsonObject = Json::object{{"value", Sb.ToString()}}; - Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsObjectId()); - CHECK(Cb["value"sv].AsObjectId() == Id); - } - - SUBCASE("iohash") - { - const uint8_t Data[] = { - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - }; - - const IoHash Hash = IoHash::HashBuffer(Data, sizeof(Data)); - - Json JsonObject = Json::object{{"value", Hash.ToHexString()}}; - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsHash()); - CHECK(Cb["value"sv].AsHash() == Hash); - - JsonObject = Json::object{{"value", "0x" + Hash.ToHexString()}}; - Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsHash()); - CHECK(Cb["value"sv].AsHash() == Hash); - } -} - -#endif - -} // namespace zen diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp deleted file mode 100644 index d4ccd434d..000000000 --- a/zencore/compactbinarybuilder.cpp +++ /dev/null @@ -1,1545 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/compactbinarybuilder.h" - -#include <zencore/compactbinarypackage.h> -#include <zencore/compactbinaryvalidation.h> -#include <zencore/endian.h> -#include <zencore/stream.h> -#include <zencore/string.h> -#include <zencore/testing.h> - -#define _USE_MATH_DEFINES -#include <math.h> - -namespace zen { - -template<typename T> -uint64_t -AddUninitialized(std::vector<T>& Vector, uint64_t Count) -{ - const uint64_t Offset = Vector.size(); - Vector.resize(Offset + Count); - return Offset; -} - -template<typename T> -uint64_t -Append(std::vector<T>& Vector, const T* Data, uint64_t Count) -{ - const uint64_t Offset = Vector.size(); - Vector.resize(Offset + Count); - - memcpy(Vector.data() + Offset, Data, sizeof(T) * Count); - - return Offset; -} - -////////////////////////////////////////////////////////////////////////// - -enum class CbWriter::StateFlags : uint8_t -{ - None = 0, - /** Whether a name has been written for the current field. */ - Name = 1 << 0, - /** Whether this state is in the process of writing a field. */ - Field = 1 << 1, - /** Whether this state is for array fields. */ - Array = 1 << 2, - /** Whether this state is for object fields. */ - Object = 1 << 3, -}; - -ENUM_CLASS_FLAGS(CbWriter::StateFlags); - -/** Whether the field type can be used in a uniform array or uniform object. */ -static constexpr bool -IsUniformType(const CbFieldType Type) -{ - if (CbFieldTypeOps::HasFieldName(Type)) - { - return true; - } - - switch (Type) - { - case CbFieldType::None: - case CbFieldType::Null: - case CbFieldType::BoolFalse: - case CbFieldType::BoolTrue: - return false; - default: - return true; - } -} - -/** Append the payload from the compact binary value to the array and return its type. */ -static inline CbFieldType -AppendCompactBinary(const CbFieldView& Value, std::vector<uint8_t>& OutData) -{ - struct FCopy : public CbFieldView - { - using CbFieldView::GetPayloadView; - using CbFieldView::GetType; - }; - const FCopy& ValueCopy = static_cast<const FCopy&>(Value); - const MemoryView SourceView = ValueCopy.GetPayloadView(); - const uint64_t TargetOffset = OutData.size(); - OutData.resize(TargetOffset + SourceView.GetSize()); - memcpy(OutData.data() + TargetOffset, SourceView.GetData(), SourceView.GetSize()); - return CbFieldTypeOps::GetType(ValueCopy.GetType()); -} - -CbWriter::CbWriter() -{ - States.emplace_back(); -} - -CbWriter::CbWriter(const int64_t InitialSize) : CbWriter() -{ - Data.reserve(InitialSize); -} - -CbWriter::~CbWriter() -{ -} - -void -CbWriter::Reset() -{ - Data.resize(0); - States.resize(0); - States.emplace_back(); -} - -CbFieldIterator -CbWriter::Save() -{ - const uint64_t Size = GetSaveSize(); - UniqueBuffer Buffer = UniqueBuffer::Alloc(Size); - const CbFieldViewIterator Output = Save(Buffer); - - SharedBuffer SharedBuf(std::move(Buffer)); - SharedBuf.MakeImmutable(); - - return CbFieldIterator::MakeRangeView(Output, SharedBuf); -} - -CbFieldViewIterator -CbWriter::Save(const MutableMemoryView Buffer) -{ - ZEN_ASSERT(States.size() == 1 && States.back().Flags == StateFlags::None); - // TEXT("It is invalid to save while there are incomplete write operations.")); - ZEN_ASSERT(Data.size() > 0); // TEXT("It is invalid to save when nothing has been written.")); - ZEN_ASSERT(Buffer.GetSize() == Data.size()); - // TEXT("Buffer is %" UINT64_FMT " bytes but %" INT64_FMT " is required."), - // Buffer.GetSize(), - // Data.Num()); - memcpy(Buffer.GetData(), Data.data(), Data.size()); - return CbFieldViewIterator::MakeRange(Buffer); -} - -void -CbWriter::Save(BinaryWriter& Writer) -{ - ZEN_ASSERT(States.size() == 1 && States.back().Flags == StateFlags::None); - // TEXT("It is invalid to save while there are incomplete write operations.")); - ZEN_ASSERT(Data.size() > 0); // TEXT("It is invalid to save when nothing has been written.")); - Writer.Write(Data.data(), Data.size()); -} - -uint64_t -CbWriter::GetSaveSize() const -{ - return Data.size(); -} - -void -CbWriter::BeginField() -{ - WriterState& State = States.back(); - if ((State.Flags & StateFlags::Field) == StateFlags::None) - { - State.Flags |= StateFlags::Field; - State.Offset = Data.size(); - Data.push_back(0); - } - else - { - ZEN_ASSERT((State.Flags & StateFlags::Name) == StateFlags::Name); - // TEXT("A new field cannot be written until the previous field '%.*hs' is finished."), - // GetActiveName().Len(), - // GetActiveName().GetData()); - } -} - -void -CbWriter::EndField(CbFieldType Type) -{ - WriterState& State = States.back(); - - if ((State.Flags & StateFlags::Name) == StateFlags::Name) - { - Type |= CbFieldType::HasFieldName; - } - else - { - ZEN_ASSERT((State.Flags & StateFlags::Object) == StateFlags::None); - // TEXT("It is invalid to write an object field without a unique non-empty name.")); - } - - if (State.Count == 0) - { - State.UniformType = Type; - } - else if (State.UniformType != Type) - { - State.UniformType = CbFieldType::None; - } - - State.Flags &= ~(StateFlags::Name | StateFlags::Field); - ++State.Count; - Data[State.Offset] = uint8_t(Type); -} - -ZEN_NOINLINE -CbWriter& -CbWriter::SetName(const std::string_view Name) -{ - WriterState& State = States.back(); - ZEN_ASSERT((State.Flags & StateFlags::Array) != StateFlags::Array); - // TEXT("It is invalid to write a name for an array field. Name '%.*hs'"), - // Name.Len(), - // Name.GetData()); - ZEN_ASSERT(!Name.empty()); - // TEXT("%s"), - //(State.Flags & EStateFlags::Object) == EStateFlags::Object - // ? TEXT("It is invalid to write an empty name for an object field. Specify a unique non-empty name.") - // : TEXT("It is invalid to write an empty name for a top-level field. Specify a name or avoid this call.")); - ZEN_ASSERT((State.Flags & (StateFlags::Name | StateFlags::Field)) == StateFlags::None); - // TEXT("A new field '%.*hs' cannot be written until the previous field '%.*hs' is finished."), - // Name.Len(), - // Name.GetData(), - // GetActiveName().Len(), - // GetActiveName().GetData()); - - BeginField(); - State.Flags |= StateFlags::Name; - const uint32_t NameLenByteCount = MeasureVarUInt(uint32_t(Name.size())); - const int64_t NameLenOffset = Data.size(); - Data.resize(NameLenOffset + NameLenByteCount); - - WriteVarUInt(uint64_t(Name.size()), Data.data() + NameLenOffset); - - const uint8_t* NamePtr = reinterpret_cast<const uint8_t*>(Name.data()); - Data.insert(Data.end(), NamePtr, NamePtr + Name.size()); - return *this; -} - -void -CbWriter::SetNameOrAddString(const std::string_view NameOrValue) -{ - // A name is only written if it would begin a new field inside of an object. - if ((States.back().Flags & (StateFlags::Name | StateFlags::Field | StateFlags::Object)) == StateFlags::Object) - { - SetName(NameOrValue); - } - else - { - AddString(NameOrValue); - } -} - -std::string_view -CbWriter::GetActiveName() const -{ - const WriterState& State = States.back(); - if ((State.Flags & StateFlags::Name) == StateFlags::Name) - { - const uint8_t* const EncodedName = Data.data() + State.Offset + sizeof(CbFieldType); - uint32_t NameLenByteCount; - const uint64_t NameLen = ReadVarUInt(EncodedName, NameLenByteCount); - const size_t ClampedNameLen = std::clamp<uint64_t>(NameLen, 0, ~uint64_t(0)); - return std::string_view(reinterpret_cast<const char*>(EncodedName + NameLenByteCount), ClampedNameLen); - } - return std::string_view(); -} - -void -CbWriter::MakeFieldsUniform(const int64_t FieldBeginOffset, const int64_t FieldEndOffset) -{ - MutableMemoryView SourceView(Data.data() + FieldBeginOffset, uint64_t(FieldEndOffset - FieldBeginOffset)); - MutableMemoryView TargetView = SourceView; - TargetView.RightChopInline(sizeof(CbFieldType)); - - while (!SourceView.IsEmpty()) - { - const uint64_t FieldSize = MeasureCompactBinary(SourceView) - sizeof(CbFieldType); - SourceView.RightChopInline(sizeof(CbFieldType)); - if (TargetView.GetData() != SourceView.GetData()) - { - memmove(TargetView.GetData(), SourceView.GetData(), FieldSize); - } - SourceView.RightChopInline(FieldSize); - TargetView.RightChopInline(FieldSize); - } - - if (!TargetView.IsEmpty()) - { - const auto EraseBegin = Data.begin() + (FieldEndOffset - TargetView.GetSize()); - const auto EraseEnd = EraseBegin + TargetView.GetSize(); - - Data.erase(EraseBegin, EraseEnd); - } -} - -void -CbWriter::AddField(const CbFieldView& Value) -{ - ZEN_ASSERT(Value.HasValue()); // , TEXT("It is invalid to write a field with no value.")); - BeginField(); - EndField(AppendCompactBinary(Value, Data)); -} - -void -CbWriter::AddField(const CbField& Value) -{ - AddField(CbFieldView(Value)); -} - -void -CbWriter::BeginObject() -{ - BeginField(); - States.push_back(WriterState()); - States.back().Flags |= StateFlags::Object; -} - -void -CbWriter::EndObject() -{ - ZEN_ASSERT(States.size() > 1 && (States.back().Flags & StateFlags::Object) == StateFlags::Object); - - // TEXT("It is invalid to end an object when an object is not at the top of the stack.")); - ZEN_ASSERT((States.back().Flags & StateFlags::Field) == StateFlags::None); - // TEXT("It is invalid to end an object until the previous field is finished.")); - - const bool bUniform = IsUniformType(States.back().UniformType); - const uint64_t Count = States.back().Count; - States.pop_back(); - - // Calculate the offset of the payload. - const WriterState& State = States.back(); - int64_t PayloadOffset = State.Offset + 1; - if ((State.Flags & StateFlags::Name) == StateFlags::Name) - { - uint32_t NameLenByteCount; - const uint64_t NameLen = ReadVarUInt(Data.data() + PayloadOffset, NameLenByteCount); - PayloadOffset += NameLen + NameLenByteCount; - } - - // Remove redundant field types for uniform objects. - if (bUniform && Count > 1) - { - MakeFieldsUniform(PayloadOffset, Data.size()); - } - - // Insert the object size. - const uint64_t Size = uint64_t(Data.size() - PayloadOffset); - const uint32_t SizeByteCount = MeasureVarUInt(Size); - Data.insert(Data.begin() + PayloadOffset, SizeByteCount, 0); - WriteVarUInt(Size, Data.data() + PayloadOffset); - - EndField(bUniform ? CbFieldType::UniformObject : CbFieldType::Object); -} - -void -CbWriter::AddObject(const CbObjectView& Value) -{ - BeginField(); - EndField(AppendCompactBinary(Value.AsFieldView(), Data)); -} - -void -CbWriter::AddObject(const CbObject& Value) -{ - AddObject(CbObjectView(Value)); -} - -ZEN_NOINLINE -void -CbWriter::BeginArray() -{ - BeginField(); - States.push_back(WriterState()); - States.back().Flags |= StateFlags::Array; -} - -void -CbWriter::EndArray() -{ - ZEN_ASSERT(States.size() > 1 && (States.back().Flags & StateFlags::Array) == StateFlags::Array); - // TEXT("Invalid attempt to end an array when an array is not at the top of the stack.")); - ZEN_ASSERT((States.back().Flags & StateFlags::Field) == StateFlags::None); - // TEXT("It is invalid to end an array until the previous field is finished.")); - const bool bUniform = IsUniformType(States.back().UniformType); - const uint64_t Count = States.back().Count; - States.pop_back(); - - // Calculate the offset of the payload. - const WriterState& State = States.back(); - int64_t PayloadOffset = State.Offset + 1; - if ((State.Flags & StateFlags::Name) == StateFlags::Name) - { - uint32_t NameLenByteCount; - const uint64_t NameLen = ReadVarUInt(Data.data() + PayloadOffset, NameLenByteCount); - PayloadOffset += NameLen + NameLenByteCount; - } - - // Remove redundant field types for uniform arrays. - if (bUniform && Count > 1) - { - MakeFieldsUniform(PayloadOffset, Data.size()); - } - - // Insert the array size and field count. - const uint32_t CountByteCount = MeasureVarUInt(Count); - const uint64_t Size = uint64_t(Data.size() - PayloadOffset) + CountByteCount; - const uint32_t SizeByteCount = MeasureVarUInt(Size); - Data.insert(Data.begin() + PayloadOffset, SizeByteCount + CountByteCount, 0); - WriteVarUInt(Size, Data.data() + PayloadOffset); - WriteVarUInt(Count, Data.data() + PayloadOffset + SizeByteCount); - - EndField(bUniform ? CbFieldType::UniformArray : CbFieldType::Array); -} - -void -CbWriter::AddArray(const CbArrayView& Value) -{ - BeginField(); - EndField(AppendCompactBinary(Value.AsFieldView(), Data)); -} - -void -CbWriter::AddArray(const CbArray& Value) -{ - AddArray(CbArrayView(Value)); -} - -void -CbWriter::AddNull() -{ - BeginField(); - EndField(CbFieldType::Null); -} - -void -CbWriter::AddBinary(const void* const Value, const uint64_t Size) -{ - const size_t SizeByteCount = MeasureVarUInt(Size); - Data.reserve(Data.size() + 1 + SizeByteCount + Size); - BeginField(); - const size_t SizeOffset = Data.size(); - Data.resize(Data.size() + SizeByteCount); - WriteVarUInt(Size, Data.data() + SizeOffset); - Data.insert(Data.end(), static_cast<const uint8_t*>(Value), static_cast<const uint8_t*>(Value) + Size); - EndField(CbFieldType::Binary); -} - -void -CbWriter::AddBinary(IoBuffer Buffer) -{ - AddBinary(Buffer.Data(), Buffer.Size()); -} - -void -CbWriter::AddBinary(SharedBuffer Buffer) -{ - AddBinary(Buffer.GetData(), Buffer.GetSize()); -} - -void -CbWriter::AddBinary(const CompositeBuffer& Buffer) -{ - AddBinary(Buffer.Flatten()); -} - -void -CbWriter::AddString(const std::string_view Value) -{ - BeginField(); - const uint64_t Size = uint64_t(Value.size()); - const uint32_t SizeByteCount = MeasureVarUInt(Size); - const int64_t Offset = Data.size(); - - Data.resize(Offset + SizeByteCount + Size); - - uint8_t* StringData = Data.data() + Offset; - WriteVarUInt(Size, StringData); - StringData += SizeByteCount; - if (Size > 0) - { - memcpy(StringData, Value.data(), Value.size() * sizeof(char)); - } - EndField(CbFieldType::String); -} - -void -CbWriter::AddString(const std::wstring_view Value) -{ - BeginField(); - ExtendableStringBuilder<128> Utf8; - WideToUtf8(Value, Utf8); - - const uint32_t Size = uint32_t(Utf8.Size()); - const uint32_t SizeByteCount = MeasureVarUInt(Size); - const int64_t Offset = Data.size(); - Data.resize(Offset + SizeByteCount + Size); - uint8_t* StringData = Data.data() + Offset; - WriteVarUInt(Size, StringData); - StringData += SizeByteCount; - if (Size > 0) - { - memcpy(reinterpret_cast<char*>(StringData), Utf8.Data(), Utf8.Size()); - } - EndField(CbFieldType::String); -} - -ZEN_NOINLINE -void -CbWriter::AddInteger(const int32_t Value) -{ - if (Value >= 0) - { - return AddInteger(uint32_t(Value)); - } - BeginField(); - const uint32_t Magnitude = ~uint32_t(Value); - const uint32_t MagnitudeByteCount = MeasureVarUInt(Magnitude); - const int64_t Offset = Data.size(); - Data.resize(Offset + MagnitudeByteCount); - WriteVarUInt(Magnitude, Data.data() + Offset); - EndField(CbFieldType::IntegerNegative); -} - -void -CbWriter::AddInteger(const int64_t Value) -{ - if (Value >= 0) - { - return AddInteger(uint64_t(Value)); - } - BeginField(); - const uint64_t Magnitude = ~uint64_t(Value); - const uint32_t MagnitudeByteCount = MeasureVarUInt(Magnitude); - const uint64_t Offset = AddUninitialized(Data, MagnitudeByteCount); - WriteVarUInt(Magnitude, Data.data() + Offset); - EndField(CbFieldType::IntegerNegative); -} - -ZEN_NOINLINE -void -CbWriter::AddInteger(const uint32_t Value) -{ - BeginField(); - const uint32_t ValueByteCount = MeasureVarUInt(Value); - const uint64_t Offset = AddUninitialized(Data, ValueByteCount); - WriteVarUInt(Value, Data.data() + Offset); - EndField(CbFieldType::IntegerPositive); -} - -ZEN_NOINLINE -void -CbWriter::AddInteger(const uint64_t Value) -{ - BeginField(); - const uint32_t ValueByteCount = MeasureVarUInt(Value); - const uint64_t Offset = AddUninitialized(Data, ValueByteCount); - WriteVarUInt(Value, Data.data() + Offset); - EndField(CbFieldType::IntegerPositive); -} - -ZEN_NOINLINE -void -CbWriter::AddFloat(const float Value) -{ - BeginField(); - const uint32_t RawValue = FromNetworkOrder(reinterpret_cast<const uint32_t&>(Value)); - Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint32_t)); - EndField(CbFieldType::Float32); -} - -ZEN_NOINLINE -void -CbWriter::AddFloat(const double Value) -{ - const float Value32 = float(Value); - if (Value == double(Value32)) - { - return AddFloat(Value32); - } - BeginField(); - const uint64_t RawValue = FromNetworkOrder(reinterpret_cast<const uint64_t&>(Value)); - Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t)); - EndField(CbFieldType::Float64); -} - -ZEN_NOINLINE -void -CbWriter::AddBool(const bool bValue) -{ - BeginField(); - EndField(bValue ? CbFieldType::BoolTrue : CbFieldType::BoolFalse); -} - -ZEN_NOINLINE -void -CbWriter::AddObjectAttachment(const IoHash& Value) -{ - BeginField(); - Append(Data, Value.Hash, sizeof Value.Hash); - EndField(CbFieldType::ObjectAttachment); -} - -ZEN_NOINLINE -void -CbWriter::AddBinaryAttachment(const IoHash& Value) -{ - BeginField(); - Append(Data, Value.Hash, sizeof Value.Hash); - EndField(CbFieldType::BinaryAttachment); -} - -ZEN_NOINLINE -void -CbWriter::AddAttachment(const CbAttachment& Attachment) -{ - BeginField(); - const IoHash& Value = Attachment.GetHash(); - Append(Data, Value.Hash, sizeof Value.Hash); - EndField(CbFieldType::BinaryAttachment); -} - -ZEN_NOINLINE -void -CbWriter::AddHash(const IoHash& Value) -{ - BeginField(); - Append(Data, Value.Hash, sizeof Value.Hash); - EndField(CbFieldType::Hash); -} - -void -CbWriter::AddUuid(const Guid& Value) -{ - const auto AppendSwappedBytes = [this](uint32_t In) { - In = FromNetworkOrder(In); - Append(Data, reinterpret_cast<const uint8_t*>(&In), sizeof In); - }; - BeginField(); - AppendSwappedBytes(Value.A); - AppendSwappedBytes(Value.B); - AppendSwappedBytes(Value.C); - AppendSwappedBytes(Value.D); - EndField(CbFieldType::Uuid); -} - -void -CbWriter::AddObjectId(const Oid& Value) -{ - BeginField(); - Append(Data, reinterpret_cast<const uint8_t*>(&Value.OidBits), sizeof Value.OidBits); - EndField(CbFieldType::ObjectId); -} - -void -CbWriter::AddDateTimeTicks(const int64_t Ticks) -{ - BeginField(); - const uint64_t RawValue = FromNetworkOrder(uint64_t(Ticks)); - Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t)); - EndField(CbFieldType::DateTime); -} - -void -CbWriter::AddDateTime(const DateTime Value) -{ - AddDateTimeTicks(Value.GetTicks()); -} - -void -CbWriter::AddTimeSpanTicks(const int64_t Ticks) -{ - BeginField(); - const uint64_t RawValue = FromNetworkOrder(uint64_t(Ticks)); - Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t)); - EndField(CbFieldType::TimeSpan); -} - -void -CbWriter::AddTimeSpan(const TimeSpan Value) -{ - AddTimeSpanTicks(Value.GetTicks()); -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -CbWriter& -operator<<(CbWriter& Writer, const DateTime Value) -{ - Writer.AddDateTime(Value); - return Writer; -} - -CbWriter& -operator<<(CbWriter& Writer, const TimeSpan Value) -{ - Writer.AddTimeSpan(Value); - return Writer; -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#if ZEN_WITH_TESTS -void -usonbuilder_forcelink() -{ -} - -// doctest::String -// toString(const DateTime&) -// { -// // TODO:implement -// return ""; -// } - -// doctest::String -// toString(const TimeSpan&) -// { -// // TODO:implement -// return ""; -// } - -TEST_CASE("usonbuilder.object") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - SUBCASE("EmptyObject") - { - Writer.BeginObject(); - Writer.EndObject(); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.IsObject() == true); - CHECK(Field.AsObjectView().CreateViewIterator().HasValue() == false); - } - - SUBCASE("NamedEmptyObject") - { - Writer.SetName("Object"sv); - Writer.BeginObject(); - Writer.EndObject(); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.IsObject() == true); - CHECK(Field.AsObjectView().CreateViewIterator().HasValue() == false); - } - - SUBCASE("BasicObject") - { - Writer.BeginObject(); - Writer.SetName("Integer"sv).AddInteger(0); - Writer.SetName("Float"sv).AddFloat(0.0f); - Writer.EndObject(); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.IsObject() == true); - - CbObjectView Object = Field.AsObjectView(); - CHECK(Object["Integer"sv].IsInteger() == true); - CHECK(Object["Float"sv].IsFloat() == true); - } - - SUBCASE("UniformObject") - { - Writer.BeginObject(); - Writer.SetName("Field1"sv).AddInteger(0); - Writer.SetName("Field2"sv).AddInteger(1); - Writer.EndObject(); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.IsObject() == true); - - CbObjectView Object = Field.AsObjectView(); - CHECK(Object["Field1"sv].IsInteger() == true); - CHECK(Object["Field2"sv].IsInteger() == true); - } -} - -TEST_CASE("usonbuilder.array") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - SUBCASE("EmptyArray") - { - Writer.BeginArray(); - Writer.EndArray(); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.IsArray() == true); - CHECK(Field.AsArrayView().Num() == 0); - } - - SUBCASE("NamedEmptyArray") - { - Writer.SetName("Array"sv); - Writer.BeginArray(); - Writer.EndArray(); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.IsArray() == true); - CHECK(Field.AsArrayView().Num() == 0); - } - - SUBCASE("BasicArray") - { - Writer.BeginArray(); - Writer.AddInteger(0); - Writer.AddFloat(0.0f); - Writer.EndArray(); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.IsArray() == true); - CbFieldViewIterator Iterator = Field.AsArrayView().CreateViewIterator(); - CHECK(Iterator.IsInteger() == true); - ++Iterator; - CHECK(Iterator.IsFloat() == true); - ++Iterator; - CHECK(Iterator.HasValue() == false); - } - - SUBCASE("UniformArray") - { - Writer.BeginArray(); - Writer.AddInteger(0); - Writer.AddInteger(1); - Writer.EndArray(); - - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.IsArray() == true); - CbFieldViewIterator Iterator = Field.AsArrayView().CreateViewIterator(); - CHECK(Iterator.IsInteger() == true); - ++Iterator; - CHECK(Iterator.IsInteger() == true); - ++Iterator; - CHECK(Iterator.HasValue() == false); - } -} - -TEST_CASE("usonbuilder.null") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - SUBCASE("Null") - { - Writer.AddNull(); - CbField Field = Writer.Save(); - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.HasName() == false); - CHECK(Field.IsNull() == true); - } - - SUBCASE("NullWithName") - { - Writer.SetName("Null"sv); - Writer.AddNull(); - CbField Field = Writer.Save(); - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - CHECK(Field.HasName() == true); - CHECK(Field.GetName().compare("Null"sv) == 0); - CHECK(Field.IsNull() == true); - } - - SUBCASE("Null Array/Object Uniformity") - { - Writer.BeginArray(); - Writer.AddNull(); - Writer.AddNull(); - Writer.AddNull(); - Writer.EndArray(); - - Writer.BeginObject(); - Writer.SetName("N1"sv).AddNull(); - Writer.SetName("N2"sv).AddNull(); - Writer.SetName("N3"sv).AddNull(); - Writer.EndObject(); - - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - } - - SUBCASE("Null with Save(Buffer)") - { - constexpr int NullCount = 3; - for (int Index = 0; Index < NullCount; ++Index) - { - Writer.AddNull(); - } - uint8_t Buffer[NullCount]{}; - MutableMemoryView BufferView(Buffer, sizeof Buffer); - CbFieldViewIterator Fields = Writer.Save(BufferView); - - CHECK(ValidateCompactBinaryRange(BufferView, CbValidateMode::All) == CbValidateError::None); - - for (int Index = 0; Index < NullCount; ++Index) - { - CHECK(Fields.IsNull() == true); - ++Fields; - } - CHECK(Fields.HasValue() == false); - } -} - -TEST_CASE("usonbuilder.binary") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; -} - -TEST_CASE("usonbuilder.string") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - SUBCASE("Empty Strings") - { - Writer.AddString(std::string_view()); - Writer.AddString(std::wstring_view()); - - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - for (CbFieldView Field : Fields) - { - CHECK(Field.HasName() == false); - CHECK(Field.IsString() == true); - CHECK(Field.AsString().empty() == true); - } - } - - SUBCASE("Test Basic Strings") - { - Writer.SetName("String"sv).AddString("Value"sv); - Writer.SetName("String"sv).AddString(L"Value"sv); - - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - for (CbFieldView Field : Fields) - { - CHECK(Field.GetName().compare("String"sv) == 0); - CHECK(Field.HasName() == true); - CHECK(Field.IsString() == true); - CHECK(Field.AsString().compare("Value"sv) == 0); - } - } - - SUBCASE("Long Strings") - { - constexpr int DotCount = 256; - StringBuilder<DotCount + 1> Dots; - for (int Index = 0; Index < DotCount; ++Index) - { - Dots.Append('.'); - } - Writer.AddString(Dots); - Writer.AddString(std::wstring().append(256, L'.')); - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - for (CbFieldView Field : Fields) - { - CHECK((Field.AsString() == std::string_view(Dots))); - } - } - - SUBCASE("Non-ASCII String") - { -# if ZEN_SIZEOF_WCHAR_T == 2 - wchar_t Value[2] = {0xd83d, 0xde00}; -# else - wchar_t Value[1] = {0x1f600}; -# endif - - Writer.AddString("\xf0\x9f\x98\x80"sv); - Writer.AddString(std::wstring_view(Value, ZEN_ARRAY_COUNT(Value))); - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - for (CbFieldView Field : Fields) - { - CHECK((Field.AsString() == "\xf0\x9f\x98\x80"sv)); - } - } -} - -TEST_CASE("usonbuilder.integer") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - auto TestInt32 = [&Writer](int32_t Value) { - Writer.Reset(); - Writer.AddInteger(Value); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - CHECK(Field.AsInt32() == Value); - CHECK(Field.HasError() == false); - }; - - auto TestUInt32 = [&Writer](uint32_t Value) { - Writer.Reset(); - Writer.AddInteger(Value); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - CHECK(Field.AsUInt32() == Value); - CHECK(Field.HasError() == false); - }; - - auto TestInt64 = [&Writer](int64_t Value) { - Writer.Reset(); - Writer.AddInteger(Value); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - CHECK(Field.AsInt64() == Value); - CHECK(Field.HasError() == false); - }; - - auto TestUInt64 = [&Writer](uint64_t Value) { - Writer.Reset(); - Writer.AddInteger(Value); - CbField Field = Writer.Save(); - - CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - CHECK(Field.AsUInt64() == Value); - CHECK(Field.HasError() == false); - }; - - TestUInt32(uint32_t(0x00)); - TestUInt32(uint32_t(0x7f)); - TestUInt32(uint32_t(0x80)); - TestUInt32(uint32_t(0xff)); - TestUInt32(uint32_t(0x0100)); - TestUInt32(uint32_t(0x7fff)); - TestUInt32(uint32_t(0x8000)); - TestUInt32(uint32_t(0xffff)); - TestUInt32(uint32_t(0x0001'0000)); - TestUInt32(uint32_t(0x7fff'ffff)); - TestUInt32(uint32_t(0x8000'0000)); - TestUInt32(uint32_t(0xffff'ffff)); - - TestUInt64(uint64_t(0x0000'0001'0000'0000)); - TestUInt64(uint64_t(0x7fff'ffff'ffff'ffff)); - TestUInt64(uint64_t(0x8000'0000'0000'0000)); - TestUInt64(uint64_t(0xffff'ffff'ffff'ffff)); - - TestInt32(int32_t(0x01)); - TestInt32(int32_t(0x80)); - TestInt32(int32_t(0x81)); - TestInt32(int32_t(0x8000)); - TestInt32(int32_t(0x8001)); - TestInt32(int32_t(0x7fff'ffff)); - TestInt32(int32_t(0x8000'0000)); - TestInt32(int32_t(0x8000'0001)); - - TestInt64(int64_t(0x0000'0001'0000'0000)); - TestInt64(int64_t(0x8000'0000'0000'0000)); - TestInt64(int64_t(0x7fff'ffff'ffff'ffff)); - TestInt64(int64_t(0x8000'0000'0000'0001)); - TestInt64(int64_t(0xffff'ffff'ffff'ffff)); -} - -TEST_CASE("usonbuilder.float") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - SUBCASE("Float32") - { - constexpr float Values[] = { - 0.0f, - 1.0f, - -1.0f, - 3.14159265358979323846f, // PI - 3.402823466e+38f, // FLT_MAX - 1.175494351e-38f // FLT_MIN - }; - - for (float Value : Values) - { - Writer.AddFloat(Value); - } - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - const float* CheckValue = Values; - for (CbFieldView Field : Fields) - { - CHECK(Field.AsFloat() == *CheckValue++); - CHECK(Field.HasError() == false); - } - } - - SUBCASE("Float64") - { - constexpr double Values[] = { - 0.0f, - 1.0f, - -1.0f, - 3.14159265358979323846, // PI - 1.9999998807907104, - 1.9999999403953552, - 3.4028234663852886e38, - 6.8056469327705771e38, - 2.2250738585072014e-308, // DBL_MIN - 1.7976931348623158e+308 // DBL_MAX - }; - - for (double Value : Values) - { - Writer.AddFloat(Value); - } - - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - const double* CheckValue = Values; - for (CbFieldView Field : Fields) - { - CHECK(Field.AsDouble() == *CheckValue++); - CHECK(Field.HasError() == false); - } - } -} - -TEST_CASE("usonbuilder.bool") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - SUBCASE("Bool") - { - Writer.AddBool(true); - Writer.AddBool(false); - - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - CHECK(Fields.AsBool() == true); - CHECK(Fields.HasError() == false); - ++Fields; - CHECK(Fields.AsBool() == false); - CHECK(Fields.HasError() == false); - ++Fields; - CHECK(Fields.HasValue() == false); - } - - SUBCASE("Bool Array/Object Uniformity") - { - Writer.BeginArray(); - Writer.AddBool(false); - Writer.AddBool(false); - Writer.AddBool(false); - Writer.EndArray(); - - Writer.BeginObject(); - Writer.SetName("B1"sv).AddBool(false); - Writer.SetName("B2"sv).AddBool(false); - Writer.SetName("B3"sv).AddBool(false); - Writer.EndObject(); - - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - } -} - -TEST_CASE("usonbuilder.usonattachment") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; -} - -TEST_CASE("usonbuilder.binaryattachment") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; -} - -TEST_CASE("usonbuilder.hash") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; -} - -TEST_CASE("usonbuilder.uuid") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; -} - -TEST_CASE("usonbuilder.datetime") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - const DateTime Values[] = {DateTime(0), DateTime(2020, 5, 13, 15, 10)}; - for (DateTime Value : Values) - { - Writer.AddDateTime(Value); - } - - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - const DateTime* CheckValue = Values; - for (CbFieldView Field : Fields) - { - CHECK(Field.AsDateTime() == *CheckValue++); - CHECK(Field.HasError() == false); - } -} - -TEST_CASE("usonbuilder.timespan") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - const TimeSpan Values[] = {TimeSpan(0), TimeSpan(1, 2, 4, 8)}; - for (TimeSpan Value : Values) - { - Writer.AddTimeSpan(Value); - } - - CbFieldIterator Fields = Writer.Save(); - - CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - - const TimeSpan* CheckValue = Values; - for (CbFieldView Field : Fields) - { - CHECK(Field.AsTimeSpan() == *CheckValue++); - CHECK(Field.HasError() == false); - } -} - -TEST_CASE("usonbuilder.complex") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - SUBCASE("complex") - { - CbObject Object; - - { - Writer.BeginObject(); - - const uint8_t LocalField[] = {uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), 1, 'I', 42}; - Writer.AddField("FieldCopy"sv, CbFieldView(LocalField)); - Writer.AddField("FieldRefCopy"sv, CbField(SharedBuffer::Clone(MakeMemoryView(LocalField)))); - - const uint8_t LocalObject[] = {uint8_t(CbFieldType::Object | CbFieldType::HasFieldName), - 1, - 'O', - 7, - uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), - 1, - 'I', - 42, - uint8_t(CbFieldType::Null | CbFieldType::HasFieldName), - 1, - 'N'}; - Writer.AddObject("ObjectCopy"sv, CbObjectView(LocalObject)); - Writer.AddObject("ObjectRefCopy"sv, CbObject(SharedBuffer::Clone(MakeMemoryView(LocalObject)))); - - const uint8_t LocalArray[] = {uint8_t(CbFieldType::UniformArray | CbFieldType::HasFieldName), - 1, - 'A', - 4, - 2, - uint8_t(CbFieldType::IntegerPositive), - 42, - 21}; - Writer.AddArray("ArrayCopy"sv, CbArrayView(LocalArray)); - Writer.AddArray("ArrayRefCopy"sv, CbArray(SharedBuffer::Clone(MakeMemoryView(LocalArray)))); - - Writer.AddNull("Null"sv); - - Writer.BeginObject("Binary"sv); - { - Writer.AddBinary("Empty"sv, MemoryView()); - Writer.AddBinary("Value"sv, MakeMemoryView("BinaryValue")); - Writer.AddBinary("LargeValue"sv, MakeMemoryView(std::wstring().append(256, L'.'))); - Writer.AddBinary("LargeRefValue"sv, SharedBuffer::Clone(MakeMemoryView(std::wstring().append(256, L'!')))); - } - Writer.EndObject(); - - Writer.BeginObject("Strings"sv); - { - Writer.AddString("AnsiString"sv, "AnsiValue"sv); - Writer.AddString("WideString"sv, std::wstring().append(256, L'.')); - Writer.AddString("EmptyAnsiString"sv, std::string_view()); - Writer.AddString("EmptyWideString"sv, std::wstring_view()); - Writer.AddString("AnsiStringLiteral", "AnsiValue"); - Writer.AddString("WideStringLiteral", L"AnsiValue"); - } - Writer.EndObject(); - - Writer.BeginArray("Integers"sv); - { - Writer.AddInteger(int32_t(-1)); - Writer.AddInteger(int64_t(-1)); - Writer.AddInteger(uint32_t(1)); - Writer.AddInteger(uint64_t(1)); - Writer.AddInteger(std::numeric_limits<int32_t>::min()); - Writer.AddInteger(std::numeric_limits<int32_t>::max()); - Writer.AddInteger(std::numeric_limits<uint32_t>::max()); - Writer.AddInteger(std::numeric_limits<int64_t>::min()); - Writer.AddInteger(std::numeric_limits<int64_t>::max()); - Writer.AddInteger(std::numeric_limits<uint64_t>::max()); - } - Writer.EndArray(); - - Writer.BeginArray("UniformIntegers"sv); - { - Writer.AddInteger(0); - Writer.AddInteger(std::numeric_limits<int32_t>::max()); - Writer.AddInteger(std::numeric_limits<uint32_t>::max()); - Writer.AddInteger(std::numeric_limits<int64_t>::max()); - Writer.AddInteger(std::numeric_limits<uint64_t>::max()); - } - Writer.EndArray(); - - Writer.AddFloat("Float32"sv, 1.0f); - Writer.AddFloat("Float64as32"sv, 2.0); - Writer.AddFloat("Float64"sv, 3.0e100); - - Writer.AddBool("False"sv, false); - Writer.AddBool("True"sv, true); - - Writer.AddObjectAttachment("ObjectAttachment"sv, IoHash()); - Writer.AddBinaryAttachment("BinaryAttachment"sv, IoHash()); - Writer.AddAttachment("Attachment"sv, CbAttachment()); - - Writer.AddHash("Hash"sv, IoHash()); - Writer.AddUuid("Uuid"sv, Guid()); - - Writer.AddDateTimeTicks("DateTimeZero"sv, 0); - Writer.AddDateTime("DateTime2020"sv, DateTime(2020, 5, 13, 15, 10)); - - Writer.AddTimeSpanTicks("TimeSpanZero"sv, 0); - Writer.AddTimeSpan("TimeSpan"sv, TimeSpan(1, 2, 4, 8)); - - Writer.BeginObject("NestedObjects"sv); - { - Writer.BeginObject("Empty"sv); - Writer.EndObject(); - - Writer.BeginObject("Null"sv); - Writer.AddNull("Null"sv); - Writer.EndObject(); - } - Writer.EndObject(); - - Writer.BeginArray("NestedArrays"sv); - { - Writer.BeginArray(); - Writer.EndArray(); - - Writer.BeginArray(); - Writer.AddNull(); - Writer.AddNull(); - Writer.AddNull(); - Writer.EndArray(); - - Writer.BeginArray(); - Writer.AddBool(false); - Writer.AddBool(false); - Writer.AddBool(false); - Writer.EndArray(); - - Writer.BeginArray(); - Writer.AddBool(true); - Writer.AddBool(true); - Writer.AddBool(true); - Writer.EndArray(); - } - Writer.EndArray(); - - Writer.BeginArray("ArrayOfObjects"sv); - { - Writer.BeginObject(); - Writer.EndObject(); - - Writer.BeginObject(); - Writer.AddNull("Null"sv); - Writer.EndObject(); - } - Writer.EndArray(); - - Writer.BeginArray("LargeArray"sv); - for (int Index = 0; Index < 256; ++Index) - { - Writer.AddInteger(Index - 128); - } - Writer.EndArray(); - - Writer.BeginArray("LargeUniformArray"sv); - for (int Index = 0; Index < 256; ++Index) - { - Writer.AddInteger(Index); - } - Writer.EndArray(); - - Writer.BeginArray("NestedUniformArray"sv); - for (int Index = 0; Index < 16; ++Index) - { - Writer.BeginArray(); - for (int Value = 0; Value < 4; ++Value) - { - Writer.AddInteger(Value); - } - Writer.EndArray(); - } - Writer.EndArray(); - - Writer.EndObject(); - Object = Writer.Save().AsObject(); - } - CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - } -} - -TEST_CASE("usonbuilder.stream") -{ - using namespace std::literals; - - FixedCbWriter<256> Writer; - - SUBCASE("basic") - { - CbObject Object; - { - Writer.BeginObject(); - - const uint8_t LocalField[] = {uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), 1, 'I', 42}; - Writer << "FieldCopy"sv << CbFieldView(LocalField); - - const uint8_t LocalObject[] = {uint8_t(CbFieldType::Object | CbFieldType::HasFieldName), - 1, - 'O', - 7, - uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), - 1, - 'I', - 42, - uint8_t(CbFieldType::Null | CbFieldType::HasFieldName), - 1, - 'N'}; - Writer << "ObjectCopy"sv << CbObjectView(LocalObject); - - const uint8_t LocalArray[] = {uint8_t(CbFieldType::UniformArray | CbFieldType::HasFieldName), - 1, - 'A', - 4, - 2, - uint8_t(CbFieldType::IntegerPositive), - 42, - 21}; - Writer << "ArrayCopy"sv << CbArrayView(LocalArray); - - Writer << "Null"sv << nullptr; - - Writer << "Strings"sv; - Writer.BeginObject(); - Writer << "AnsiString"sv - << "AnsiValue"sv - << "AnsiStringLiteral"sv - << "AnsiValue" - << "WideString"sv << L"WideValue"sv << "WideStringLiteral"sv << L"WideValue"; - Writer.EndObject(); - - Writer << "Integers"sv; - Writer.BeginArray(); - Writer << int32_t(-1) << int64_t(-1) << uint32_t(1) << uint64_t(1); - Writer.EndArray(); - - Writer << "Float32"sv << 1.0f; - Writer << "Float64"sv << 2.0; - - Writer << "False"sv << false << "True"sv << true; - - Writer << "Attachment"sv << CbAttachment(); - - Writer << "Hash"sv << IoHash(); - Writer << "Uuid"sv << Guid(); - - Writer << "DateTime"sv << DateTime(2020, 5, 13, 15, 10); - Writer << "TimeSpan"sv << TimeSpan(1, 2, 4, 8); - - Writer << "LiteralName" << nullptr; - - Writer.EndObject(); - Object = Writer.Save().AsObject(); - } - - CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None); - } -} -#endif - -} // namespace zen diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp deleted file mode 100644 index a4fa38a1d..000000000 --- a/zencore/compactbinarypackage.cpp +++ /dev/null @@ -1,1350 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/compactbinarypackage.h" -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryvalidation.h> -#include <zencore/endian.h> -#include <zencore/stream.h> -#include <zencore/testing.h> - -namespace zen { - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -CbAttachment::CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash) : CbAttachment(InValue.MakeOwned(), Hash) -{ -} - -CbAttachment::CbAttachment(const SharedBuffer& InValue) : CbAttachment(CompositeBuffer(InValue)) -{ -} - -CbAttachment::CbAttachment(const SharedBuffer& InValue, const IoHash& InHash) : CbAttachment(CompositeBuffer(InValue), InHash) -{ -} - -CbAttachment::CbAttachment(const CompositeBuffer& InValue) -: Hash(InValue.IsNull() ? IoHash::Zero : IoHash::HashBuffer(InValue)) -, Value(InValue) -{ - if (std::get<CompositeBuffer>(Value).IsNull()) - { - Value.emplace<std::nullptr_t>(); - } -} - -CbAttachment::CbAttachment(CompositeBuffer&& InValue) -: Hash(InValue.IsNull() ? IoHash::Zero : IoHash::HashBuffer(InValue)) -, Value(std::move(InValue)) - -{ - if (std::get<CompositeBuffer>(Value).IsNull()) - { - Value.emplace<std::nullptr_t>(); - } -} - -CbAttachment::CbAttachment(CompositeBuffer&& InValue, const IoHash& InHash) : Hash(InHash), Value(InValue) -{ - if (std::get<CompositeBuffer>(Value).IsNull()) - { - Value.emplace<std::nullptr_t>(); - } -} - -CbAttachment::CbAttachment(CompressedBuffer&& InValue, const IoHash& InHash) : Hash(InHash), Value(InValue) -{ - if (std::get<CompressedBuffer>(Value).IsNull()) - { - Value.emplace<std::nullptr_t>(); - } -} - -CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash) -{ - auto SetValue = [&](const CbObject& ValueToSet) { - if (InHash) - { - Value.emplace<CbObject>(ValueToSet); - Hash = *InHash; - } - else - { - Value.emplace<CbObject>(ValueToSet); - Hash = ValueToSet.GetHash(); - } - }; - - MemoryView View; - if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View)) - { - SetValue(CbObject::Clone(InValue)); - } - else - { - SetValue(InValue); - } -} - -bool -CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator) -{ - BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); - - return TryLoad(Reader, Allocator); -} - -bool -CbAttachment::TryLoad(CbFieldIterator& Fields) -{ - if (const CbObjectView ObjectView = Fields.AsObjectView(); !Fields.HasError()) - { - // Is a null object or object not prefixed with a precomputed hash value - Value.emplace<CbObject>(CbObject(ObjectView, Fields.GetOuterBuffer())); - Hash = ObjectView.GetHash(); - ++Fields; - } - else if (const IoHash ObjectAttachmentHash = Fields.AsObjectAttachment(); !Fields.HasError()) - { - // Is an object - ++Fields; - const CbObjectView InnerObjectView = Fields.AsObjectView(); - if (Fields.HasError()) - { - return false; - } - Value.emplace<CbObject>(CbObject(InnerObjectView, Fields.GetOuterBuffer())); - Hash = ObjectAttachmentHash; - ++Fields; - } - else if (const IoHash BinaryAttachmentHash = Fields.AsBinaryAttachment(); !Fields.HasError()) - { - // Is an uncompressed binary blob - ++Fields; - MemoryView BinaryView = Fields.AsBinaryView(); - if (Fields.HasError()) - { - return false; - } - Value.emplace<CompositeBuffer>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())); - Hash = BinaryAttachmentHash; - ++Fields; - } - else if (MemoryView BinaryView = Fields.AsBinaryView(); !Fields.HasError()) - { - if (BinaryView.GetSize() > 0) - { - // Is a compressed binary blob - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = - CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), RawHash, RawSize).MakeOwned(); - Value.emplace<CompressedBuffer>(Compressed); - Hash = RawHash; - ++Fields; - } - else - { - // Is an uncompressed empty binary blob - Value.emplace<CompositeBuffer>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())); - Hash = IoHash::HashBuffer(nullptr, 0); - ++Fields; - } - } - else - { - return false; - } - - return true; -} - -static bool -TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Field, BinaryReader& Reader, BufferAllocator Allocator) -{ - if (const CbObjectView ObjectView = Field.AsObjectView(); !Field.HasError()) - { - // Is a null object or object not prefixed with a precomputed hash value - TargetAttachment = CbAttachment(CbObject(ObjectView, std::move(Field)), ObjectView.GetHash()); - } - else if (const IoHash ObjectAttachmentHash = Field.AsObjectAttachment(); !Field.HasError()) - { - // Is an object - Field = LoadCompactBinary(Reader, Allocator); - if (!Field.IsObject()) - { - return false; - } - TargetAttachment = CbAttachment(std::move(Field).AsObject(), ObjectAttachmentHash); - } - else if (const IoHash BinaryAttachmentHash = Field.AsBinaryAttachment(); !Field.HasError()) - { - // Is an uncompressed binary blob - Field = LoadCompactBinary(Reader, Allocator); - SharedBuffer Buffer = Field.AsBinary(); - if (Field.HasError()) - { - return false; - } - TargetAttachment = CbAttachment(CompositeBuffer(Buffer), BinaryAttachmentHash); - } - else if (SharedBuffer Buffer = Field.AsBinary(); !Field.HasError()) - { - if (Buffer.GetSize() > 0) - { - // Is a compressed binary blob - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer), RawHash, RawSize); - TargetAttachment = CbAttachment(Compressed, RawHash); - } - else - { - // Is an uncompressed empty binary blob - TargetAttachment = CbAttachment(CompositeBuffer(Buffer), IoHash::HashBuffer(nullptr, 0)); - } - } - else - { - return false; - } - - return true; -} - -bool -CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator) -{ - CbField Field = LoadCompactBinary(Reader, Allocator); - return TryLoad_ArchiveFieldIntoAttachment(*this, std::move(Field), Reader, Allocator); -} - -void -CbAttachment::Save(CbWriter& Writer) const -{ - if (const CbObject* Object = std::get_if<CbObject>(&Value)) - { - if (*Object) - { - Writer.AddObjectAttachment(Hash); - } - Writer.AddObject(*Object); - } - else if (const CompositeBuffer* Binary = std::get_if<CompositeBuffer>(&Value)) - { - if (Binary->GetSize() > 0) - { - Writer.AddBinaryAttachment(Hash); - } - Writer.AddBinary(*Binary); - } - else if (const CompressedBuffer* Compressed = std::get_if<CompressedBuffer>(&Value)) - { - Writer.AddBinary(Compressed->GetCompressed()); - } -} - -void -CbAttachment::Save(BinaryWriter& Writer) const -{ - CbWriter TempWriter; - Save(TempWriter); - TempWriter.Save(Writer); -} - -bool -CbAttachment::IsNull() const -{ - return std::holds_alternative<std::nullptr_t>(Value); -} - -bool -CbAttachment::IsBinary() const -{ - return std::holds_alternative<CompositeBuffer>(Value); -} - -bool -CbAttachment::IsCompressedBinary() const -{ - return std::holds_alternative<CompressedBuffer>(Value); -} - -bool -CbAttachment::IsObject() const -{ - return std::holds_alternative<CbObject>(Value); -} - -IoHash -CbAttachment::GetHash() const -{ - return Hash; -} - -CompositeBuffer -CbAttachment::AsCompositeBinary() const -{ - if (const CompositeBuffer* BinValue = std::get_if<CompositeBuffer>(&Value)) - { - return *BinValue; - } - - return CompositeBuffer::Null; -} - -SharedBuffer -CbAttachment::AsBinary() const -{ - if (const CompositeBuffer* BinValue = std::get_if<CompositeBuffer>(&Value)) - { - return BinValue->Flatten(); - } - - return {}; -} - -CompressedBuffer -CbAttachment::AsCompressedBinary() const -{ - if (const CompressedBuffer* CompValue = std::get_if<CompressedBuffer>(&Value)) - { - return *CompValue; - } - - return CompressedBuffer::Null; -} - -/** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ -CbObject -CbAttachment::AsObject() const -{ - if (const CbObject* ObjectValue = std::get_if<CbObject>(&Value)) - { - return *ObjectValue; - } - - return {}; -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -void -CbPackage::SetObject(CbObject InObject, const IoHash* InObjectHash, AttachmentResolver* InResolver) -{ - if (InObject) - { - Object = InObject.IsOwned() ? std::move(InObject) : CbObject::Clone(InObject); - if (InObjectHash) - { - ObjectHash = *InObjectHash; - ZEN_ASSERT_SLOW(ObjectHash == Object.GetHash()); - } - else - { - ObjectHash = Object.GetHash(); - } - if (InResolver) - { - GatherAttachments(Object, *InResolver); - } - } - else - { - Object.Reset(); - ObjectHash = IoHash::Zero; - } -} - -void -CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver) -{ - if (!Attachment.IsNull()) - { - auto It = std::lower_bound(begin(Attachments), end(Attachments), Attachment); - if (It != Attachments.end() && *It == Attachment) - { - CbAttachment& Existing = *It; - Existing = Attachment; - } - else - { - Attachments.insert(It, Attachment); - } - - if (Attachment.IsObject() && Resolver) - { - GatherAttachments(Attachment.AsObject(), *Resolver); - } - } -} - -void -CbPackage::AddAttachments(std::span<const CbAttachment> InAttachments) -{ - if (InAttachments.empty()) - { - return; - } - // Assume we have no duplicates! - Attachments.insert(Attachments.end(), InAttachments.begin(), InAttachments.end()); - std::sort(Attachments.begin(), Attachments.end()); - ZEN_ASSERT_SLOW(std::unique(Attachments.begin(), Attachments.end()) == Attachments.end()); -} - -int32_t -CbPackage::RemoveAttachment(const IoHash& Hash) -{ - return gsl::narrow_cast<int32_t>( - std::erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; })); -} - -bool -CbPackage::Equals(const CbPackage& Package) const -{ - return ObjectHash == Package.ObjectHash && Attachments == Package.Attachments; -} - -const CbAttachment* -CbPackage::FindAttachment(const IoHash& Hash) const -{ - auto It = std::find_if(begin(Attachments), end(Attachments), [&Hash](const CbAttachment& Attachment) -> bool { - return Attachment.GetHash() == Hash; - }); - - if (It == end(Attachments)) - return nullptr; - - return &*It; -} - -void -CbPackage::GatherAttachments(const CbObject& Value, AttachmentResolver Resolver) -{ - Value.IterateAttachments([this, &Resolver](CbFieldView Field) { - const IoHash& Hash = Field.AsAttachment(); - - if (SharedBuffer Buffer = Resolver(Hash)) - { - if (Field.IsObjectAttachment()) - { - AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash), &Resolver); - } - else - { - AddAttachment(CbAttachment(std::move(Buffer))); - } - } - }); -} - -bool -CbPackage::TryLoad(IoBuffer InBuffer, BufferAllocator Allocator, AttachmentResolver* Mapper) -{ - BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); - - return TryLoad(Reader, Allocator, Mapper); -} - -bool -CbPackage::TryLoad(CbFieldIterator& Fields) -{ - *this = CbPackage(); - - while (Fields) - { - if (Fields.IsNull()) - { - ++Fields; - break; - } - else if (IoHash Hash = Fields.AsHash(); !Fields.HasError() && !Fields.IsAttachment()) - { - ++Fields; - CbObjectView ObjectView = Fields.AsObjectView(); - if (Fields.HasError() || Hash != ObjectView.GetHash()) - { - return false; - } - Object = CbObject(ObjectView, Fields.GetOuterBuffer()); - Object.MakeOwned(); - ObjectHash = Hash; - ++Fields; - } - else - { - CbAttachment Attachment; - if (!Attachment.TryLoad(Fields)) - { - return false; - } - AddAttachment(Attachment); - } - } - return true; -} - -bool -CbPackage::TryLoad(BinaryReader& Reader, BufferAllocator Allocator, AttachmentResolver* Mapper) -{ - // TODO: this needs to re-grow the ability to accept a reference to an attachment which is - // not embedded - - ZEN_UNUSED(Mapper); - -#if 1 - *this = CbPackage(); - for (;;) - { - CbField Field = LoadCompactBinary(Reader, Allocator); - if (!Field) - { - return false; - } - - if (Field.IsNull()) - { - return true; - } - else if (IoHash Hash = Field.AsHash(); !Field.HasError() && !Field.IsAttachment()) - { - Field = LoadCompactBinary(Reader, Allocator); - CbObjectView ObjectView = Field.AsObjectView(); - if (Field.HasError() || Hash != ObjectView.GetHash()) - { - return false; - } - Object = CbObject(ObjectView, Field.GetOuterBuffer()); - ObjectHash = Hash; - } - else - { - CbAttachment Attachment; - if (!TryLoad_ArchiveFieldIntoAttachment(Attachment, std::move(Field), Reader, Allocator)) - { - return false; - } - AddAttachment(Attachment); - } - } -#else - uint8_t StackBuffer[64]; - const auto StackAllocator = [&Allocator, &StackBuffer](uint64_t Size) -> UniqueBuffer { - if (Size <= sizeof(StackBuffer)) - { - return UniqueBuffer::MakeMutableView(StackBuffer, Size); - } - - return Allocator(Size); - }; - - *this = CbPackage(); - - for (;;) - { - CbField ValueField = LoadCompactBinary(Reader, StackAllocator); - if (!ValueField) - { - return false; - } - if (ValueField.IsNull()) - { - return true; - } - else if (ValueField.IsBinary()) - { - const MemoryView View = ValueField.AsBinaryView(); - if (View.GetSize() > 0) - { - SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer()).MakeOwned(); - CbField HashField = LoadCompactBinary(Reader, StackAllocator); - const IoHash& Hash = HashField.AsAttachment(); - ZEN_ASSERT(!HashField.HasError(), "Attachments must be a non-empty binary value with a content hash."); - if (HashField.IsObjectAttachment()) - { - AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash)); - } - else - { - AddAttachment(CbAttachment(std::move(Buffer), Hash)); - } - } - } - else if (ValueField.IsHash()) - { - const IoHash Hash = ValueField.AsHash(); - - ZEN_ASSERT(Mapper); - - AddAttachment(CbAttachment((*Mapper)(Hash), Hash)); - } - else - { - Object = ValueField.AsObject(); - if (ValueField.HasError()) - { - return false; - } - Object.MakeOwned(); - if (Object) - { - CbField HashField = LoadCompactBinary(Reader, StackAllocator); - ObjectHash = HashField.AsObjectAttachment(); - if (HashField.HasError() || Object.GetHash() != ObjectHash) - { - return false; - } - } - else - { - Object.Reset(); - } - } - } -#endif -} - -void -CbPackage::Save(CbWriter& Writer) const -{ - if (Object) - { - Writer.AddHash(ObjectHash); - Writer.AddObject(Object); - } - for (const CbAttachment& Attachment : Attachments) - { - Attachment.Save(Writer); - } - Writer.AddNull(); -} - -void -CbPackage::Save(BinaryWriter& StreamWriter) const -{ - CbWriter Writer; - Save(Writer); - Writer.Save(StreamWriter); -} - -////////////////////////////////////////////////////////////////////////// -// -// Legacy package serialization support -// - -namespace legacy { - - void SaveCbAttachment(const CbAttachment& Attachment, CbWriter& Writer) - { - if (Attachment.IsObject()) - { - CbObject Object = Attachment.AsObject(); - Writer.AddBinary(Object.GetBuffer()); - if (Object) - { - Writer.AddObjectAttachment(Attachment.GetHash()); - } - } - else if (Attachment.IsBinary()) - { - Writer.AddBinary(Attachment.AsBinary()); - Writer.AddBinaryAttachment(Attachment.GetHash()); - } - else if (Attachment.IsCompressedBinary()) - { - Writer.AddBinary(Attachment.AsCompressedBinary().GetCompressed()); - Writer.AddBinaryAttachment(Attachment.GetHash()); - } - else if (Attachment.IsNull()) - { - Writer.AddBinary(MemoryView()); - } - else - { - ZEN_NOT_IMPLEMENTED("Compressed binary is not supported in this serialization format"); - } - } - - void SaveCbPackage(const CbPackage& Package, CbWriter& Writer) - { - if (const CbObject& RootObject = Package.GetObject()) - { - Writer.AddObject(RootObject); - Writer.AddObjectAttachment(Package.GetObjectHash()); - } - for (const CbAttachment& Attachment : Package.GetAttachments()) - { - SaveCbAttachment(Attachment, Writer); - } - Writer.AddNull(); - } - - void SaveCbPackage(const CbPackage& Package, BinaryWriter& Ar) - { - CbWriter Writer; - SaveCbPackage(Package, Writer); - Writer.Save(Ar); - } - - bool TryLoadCbPackage(CbPackage& Package, IoBuffer InBuffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) - { - BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); - - return TryLoadCbPackage(Package, Reader, Allocator, Mapper); - } - - bool TryLoadCbPackage(CbPackage& Package, BinaryReader& Reader, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) - { - Package = CbPackage(); - for (;;) - { - CbField ValueField = LoadCompactBinary(Reader, Allocator); - if (!ValueField) - { - return false; - } - if (ValueField.IsNull()) - { - return true; - } - if (ValueField.IsBinary()) - { - const MemoryView View = ValueField.AsBinaryView(); - if (View.GetSize() > 0) - { - SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer()).MakeOwned(); - CbField HashField = LoadCompactBinary(Reader, Allocator); - const IoHash& Hash = HashField.AsAttachment(); - if (HashField.HasError()) - { - return false; - } - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, RawHash, RawSize)) - { - if (RawHash != Hash) - { - return false; - } - Package.AddAttachment(CbAttachment(Compressed, Hash)); - } - else - { - if (IoHash::HashBuffer(Buffer) != Hash) - { - return false; - } - if (HashField.IsObjectAttachment()) - { - Package.AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash)); - } - else - { - Package.AddAttachment(CbAttachment(CompositeBuffer(std::move(Buffer)), Hash)); - } - } - } - } - else if (ValueField.IsHash()) - { - const IoHash Hash = ValueField.AsHash(); - - ZEN_ASSERT(Mapper); - if (SharedBuffer AttachmentData = (*Mapper)(Hash)) - { - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData, RawHash, RawSize)) - { - if (RawHash != Hash) - { - return false; - } - Package.AddAttachment(CbAttachment(Compressed, Hash)); - } - else - { - const CbValidateError ValidationResult = ValidateCompactBinary(AttachmentData.GetView(), CbValidateMode::All); - if (ValidationResult != CbValidateError::None) - { - return false; - } - Package.AddAttachment(CbAttachment(CbObject(std::move(AttachmentData)), Hash)); - } - } - } - else - { - CbObject Object = ValueField.AsObject(); - if (ValueField.HasError()) - { - return false; - } - - if (Object) - { - CbField HashField = LoadCompactBinary(Reader, Allocator); - IoHash ObjectHash = HashField.AsObjectAttachment(); - if (HashField.HasError() || Object.GetHash() != ObjectHash) - { - return false; - } - Package.SetObject(Object, ObjectHash); - } - } - } - } - -} // namespace legacy - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#if ZEN_WITH_TESTS - -void -usonpackage_forcelink() -{ -} - -TEST_CASE("usonpackage") -{ - using namespace std::literals; - - const auto TestSaveLoadValidate = [&](const char* Test, const CbAttachment& Attachment) { - ZEN_UNUSED(Test); - - CbWriter Writer; - Attachment.Save(Writer); - CbFieldIterator Fields = Writer.Save(); - - BinaryWriter StreamWriter; - Attachment.Save(StreamWriter); - - CHECK(MakeMemoryView(StreamWriter).EqualBytes(Fields.GetRangeBuffer().GetView())); - CHECK(ValidateCompactBinaryRange(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None); - CHECK(ValidateObjectAttachment(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None); - - CbAttachment FromFields; - FromFields.TryLoad(Fields); - CHECK(!bool(Fields)); - CHECK(FromFields == Attachment); - - CbAttachment FromArchive; - BinaryReader Reader(MakeMemoryView(StreamWriter)); - FromArchive.TryLoad(Reader); - CHECK(Reader.CurrentOffset() == Reader.Size()); - CHECK(FromArchive == Attachment); - }; - - SUBCASE("Empty Attachment") - { - CbAttachment Attachment; - CHECK(Attachment.IsNull()); - CHECK_FALSE(bool(Attachment)); - CHECK_FALSE(bool(Attachment.AsBinary())); - CHECK_FALSE(bool(Attachment.AsObject())); - CHECK_FALSE(Attachment.IsBinary()); - CHECK_FALSE(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash::Zero); - } - - SUBCASE("Binary Attachment") - { - const SharedBuffer Buffer = SharedBuffer::Clone(MakeMemoryView<uint8_t>({0, 1, 2, 3})); - CbAttachment Attachment(Buffer); - CHECK_FALSE(Attachment.IsNull()); - CHECK(bool(Attachment)); - CHECK(Attachment.AsBinary() == Buffer); - CHECK_FALSE(bool(Attachment.AsObject())); - CHECK(Attachment.IsBinary()); - CHECK_FALSE(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash::HashBuffer(Buffer)); - TestSaveLoadValidate("Binary", Attachment); - } - - SUBCASE("Object Attachment") - { - CbWriter Writer; - Writer.BeginObject(); - Writer << "Name"sv << 42; - Writer.EndObject(); - CbObject Object = Writer.Save().AsObject(); - CbAttachment Attachment(Object); - - CHECK_FALSE(Attachment.IsNull()); - CHECK(bool(Attachment)); - CHECK(Attachment.AsBinary() == SharedBuffer()); - CHECK(Attachment.AsObject().Equals(Object)); - CHECK_FALSE(Attachment.IsBinary()); - CHECK(Attachment.IsObject()); - CHECK(Attachment.GetHash() == Object.GetHash()); - TestSaveLoadValidate("Object", Attachment); - } - - SUBCASE("Binary View") - { - const uint8_t Value[]{0, 1, 2, 3}; - SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value)); - CbAttachment Attachment(Buffer); - CHECK_FALSE(Attachment.IsNull()); - CHECK(bool(Attachment)); - CHECK(Attachment.AsBinary().GetView().EqualBytes(Buffer.GetView())); - CHECK_FALSE(bool(Attachment.AsObject())); - CHECK(Attachment.IsBinary()); - CHECK_FALSE(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash::HashBuffer(Buffer)); - } - - SUBCASE("Object View") - { - CbWriter Writer; - Writer.BeginObject(); - Writer << "Name"sv << 42; - Writer.EndObject(); - CbObject Object = Writer.Save().AsObject(); - CbObject ObjectView = CbObject::MakeView(Object); - CbAttachment Attachment(ObjectView); - - CHECK_FALSE(Attachment.IsNull()); - CHECK(bool(Attachment)); - - CHECK(Attachment.AsBinary() != ObjectView.GetBuffer()); - CHECK(Attachment.AsObject().Equals(Object)); - CHECK_FALSE(Attachment.IsBinary()); - CHECK(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash(Object.GetHash())); - } - - SUBCASE("Binary Load from View") - { - const uint8_t Value[]{0, 1, 2, 3}; - const SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value)); - CbAttachment Attachment(Buffer); - - CbWriter Writer; - Attachment.Save(Writer); - CbFieldIterator Fields = Writer.Save(); - CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); - Attachment.TryLoad(FieldsView); - - CHECK_FALSE(Attachment.IsNull()); - CHECK(bool(Attachment)); - CHECK_FALSE(FieldsView.GetRangeBuffer().GetView().Contains(Attachment.AsBinary().GetView())); - CHECK(Attachment.AsBinary().GetView().EqualBytes(Buffer.GetView())); - CHECK_FALSE(Attachment.AsObject()); - CHECK(Attachment.IsBinary()); - CHECK_FALSE(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash::HashBuffer(MakeMemoryView(Value))); - } - - SUBCASE("Object Load from View") - { - CbWriter ValueWriter; - ValueWriter.BeginObject(); - ValueWriter << "Name"sv << 42; - ValueWriter.EndObject(); - const CbObject Value = ValueWriter.Save().AsObject(); - - CHECK(ValidateCompactBinaryRange(Value.GetView(), CbValidateMode::All) == CbValidateError::None); - CbAttachment Attachment(Value); - - CbWriter Writer; - Attachment.Save(Writer); - CbFieldIterator Fields = Writer.Save(); - CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); - - Attachment.TryLoad(FieldsView); - MemoryView View; - - CHECK_FALSE(Attachment.IsNull()); - CHECK(bool(Attachment)); - CHECK(Attachment.AsBinary().GetView().EqualBytes(MemoryView())); - CHECK_FALSE((!Attachment.AsObject().TryGetSerializedView(View) || FieldsView.GetOuterBuffer().GetView().Contains(View))); - CHECK_FALSE(Attachment.IsBinary()); - CHECK(Attachment.IsObject()); - CHECK(Attachment.GetHash() == Value.GetHash()); - } - - SUBCASE("Binary Null") - { - const CbAttachment Attachment(SharedBuffer{}); - - CHECK(Attachment.IsNull()); - CHECK_FALSE(Attachment.IsBinary()); - CHECK_FALSE(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash::Zero); - } - - SUBCASE("Binary Empty") - { - const CbAttachment Attachment(UniqueBuffer::Alloc(0).MoveToShared()); - - CHECK_FALSE(Attachment.IsNull()); - CHECK(Attachment.IsBinary()); - CHECK_FALSE(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash::HashBuffer(SharedBuffer{})); - } - - SUBCASE("Object Empty") - { - const CbAttachment Attachment(CbObject{}); - - CHECK_FALSE(Attachment.IsNull()); - CHECK_FALSE(Attachment.IsBinary()); - CHECK(Attachment.IsObject()); - CHECK(Attachment.GetHash() == CbObject().GetHash()); - } -} - -TEST_CASE("usonpackage.serialization") -{ - using namespace std::literals; - - const auto TestSaveLoadValidate = [&](const char* Test, CbPackage& InOutPackage) { - ZEN_UNUSED(Test); - - CbWriter Writer; - InOutPackage.Save(Writer); - CbFieldIterator Fields = Writer.Save(); - - BinaryWriter MemStream; - InOutPackage.Save(MemStream); - - CHECK(MakeMemoryView(MemStream).EqualBytes(Fields.GetRangeBuffer().GetView())); - CHECK(ValidateCompactBinaryRange(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); - CHECK(ValidateCompactBinaryPackage(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); - - CbPackage FromFields; - FromFields.TryLoad(Fields); - CHECK_FALSE(bool(Fields)); - CHECK(FromFields == InOutPackage); - - CbPackage FromArchive; - BinaryReader ReadAr(MakeMemoryView(MemStream)); - FromArchive.TryLoad(ReadAr); - CHECK(ReadAr.CurrentOffset() == ReadAr.Size()); - CHECK(FromArchive == InOutPackage); - InOutPackage = FromArchive; - }; - - SUBCASE("Empty") - { - CbPackage Package; - CHECK(Package.IsNull()); - CHECK_FALSE(bool(Package)); - CHECK(Package.GetAttachments().size() == 0); - TestSaveLoadValidate("Empty", Package); - } - - SUBCASE("Object Only") - { - CbWriter Writer; - Writer.BeginObject(); - Writer << "Field" << 42; - Writer.EndObject(); - - const CbObject Object = Writer.Save().AsObject(); - CbPackage Package(Object); - CHECK_FALSE(Package.IsNull()); - CHECK(bool(Package)); - CHECK(Package.GetAttachments().size() == 0); - CHECK(Package.GetObject().GetOuterBuffer() == Object.GetOuterBuffer()); - CHECK(Package.GetObject()["Field"].AsInt32() == 42); - CHECK(Package.GetObjectHash() == Package.GetObject().GetHash()); - TestSaveLoadValidate("Object", Package); - } - - // Object View Only - { - CbWriter Writer; - Writer.BeginObject(); - Writer << "Field" << 42; - Writer.EndObject(); - - const CbObject Object = Writer.Save().AsObject(); - CbPackage Package(CbObject::MakeView(Object)); - CHECK_FALSE(Package.IsNull()); - CHECK(bool(Package)); - CHECK(Package.GetAttachments().size() == 0); - CHECK(Package.GetObject().GetOuterBuffer() != Object.GetOuterBuffer()); - CHECK(Package.GetObject()["Field"].AsInt32() == 42); - CHECK(Package.GetObjectHash() == Package.GetObject().GetHash()); - TestSaveLoadValidate("Object", Package); - } - - // Attachment Only - { - CbObject Object1; - { - CbWriter Writer; - Writer.BeginObject(); - Writer << "Field1" << 42; - Writer.EndObject(); - Object1 = Writer.Save().AsObject(); - } - CbObject Object2; - { - CbWriter Writer; - Writer.BeginObject(); - Writer << "Field2" << 42; - Writer.EndObject(); - Object2 = Writer.Save().AsObject(); - } - - CbPackage Package; - Package.AddAttachment(CbAttachment(Object1)); - Package.AddAttachment(CbAttachment(Object2.GetBuffer())); - - CHECK_FALSE(Package.IsNull()); - CHECK(bool(Package)); - CHECK(Package.GetAttachments().size() == 2); - CHECK(Package.GetObject().Equals(CbObject())); - CHECK(Package.GetObjectHash() == IoHash::Zero); - TestSaveLoadValidate("Attachments", Package); - - const CbAttachment* const Object1Attachment = Package.FindAttachment(Object1.GetHash()); - const CbAttachment* const Object2Attachment = Package.FindAttachment(Object2.GetHash()); - - CHECK((Object1Attachment && Object1Attachment->AsObject().Equals(Object1))); - CHECK((Object2Attachment && Object2Attachment->AsBinary().GetView().EqualBytes(Object2.GetBuffer().GetView()))); - - SharedBuffer Object1ClonedBuffer = SharedBuffer::Clone(Object1.GetOuterBuffer()); - Package.AddAttachment(CbAttachment(Object1ClonedBuffer)); - Package.AddAttachment(CbAttachment(CbObject::Clone(Object2))); - - CHECK(Package.GetAttachments().size() == 2); - CHECK(Package.FindAttachment(Object1.GetHash()) == Object1Attachment); - CHECK(Package.FindAttachment(Object2.GetHash()) == Object2Attachment); - - CHECK((Object1Attachment && Object1Attachment->AsBinary() == Object1ClonedBuffer)); - CHECK((Object2Attachment && Object2Attachment->AsObject().Equals(Object2))); - - CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); - } - - // Shared Values - const uint8_t Level4Values[]{0, 1, 2, 3}; - SharedBuffer Level4 = SharedBuffer::MakeView(MakeMemoryView(Level4Values)); - const IoHash Level4Hash = IoHash::HashBuffer(Level4); - - CbObject Level3; - { - CbWriter Writer; - Writer.BeginObject(); - Writer.AddBinaryAttachment("Level4", Level4Hash); - Writer.EndObject(); - Level3 = Writer.Save().AsObject(); - } - const IoHash Level3Hash = Level3.GetHash(); - - CbObject Level2; - { - CbWriter Writer; - Writer.BeginObject(); - Writer.AddObjectAttachment("Level3", Level3Hash); - Writer.EndObject(); - Level2 = Writer.Save().AsObject(); - } - const IoHash Level2Hash = Level2.GetHash(); - - CbObject Level1; - { - CbWriter Writer; - Writer.BeginObject(); - Writer.AddObjectAttachment("Level2", Level2Hash); - Writer.EndObject(); - Level1 = Writer.Save().AsObject(); - } - const IoHash Level1Hash = Level1.GetHash(); - - const auto Resolver = [&Level2, &Level2Hash, &Level3, &Level3Hash, &Level4, &Level4Hash](const IoHash& Hash) -> SharedBuffer { - return Hash == Level2Hash ? Level2.GetOuterBuffer() - : Hash == Level3Hash ? Level3.GetOuterBuffer() - : Hash == Level4Hash ? Level4 - : SharedBuffer(); - }; - - // Object + Attachments - { - CbPackage Package; - Package.SetObject(Level1, Level1Hash, Resolver); - - CHECK_FALSE(Package.IsNull()); - CHECK(bool(Package)); - CHECK(Package.GetAttachments().size() == 3); - CHECK(Package.GetObject().GetBuffer() == Level1.GetBuffer()); - CHECK(Package.GetObjectHash() == Level1Hash); - TestSaveLoadValidate("Object+Attachments", Package); - - const CbAttachment* const Level2Attachment = Package.FindAttachment(Level2Hash); - const CbAttachment* const Level3Attachment = Package.FindAttachment(Level3Hash); - const CbAttachment* const Level4Attachment = Package.FindAttachment(Level4Hash); - CHECK((Level2Attachment && Level2Attachment->AsObject().Equals(Level2))); - CHECK((Level3Attachment && Level3Attachment->AsObject().Equals(Level3))); - REQUIRE(Level4Attachment); - CHECK(Level4Attachment->AsBinary() != Level4); - CHECK(Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView())); - - CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); - - const CbPackage PackageCopy = Package; - CHECK(PackageCopy == Package); - - CHECK(Package.RemoveAttachment(Level1Hash) == 0); - CHECK(Package.RemoveAttachment(Level2Hash) == 1); - CHECK(Package.RemoveAttachment(Level3Hash) == 1); - CHECK(Package.RemoveAttachment(Level4Hash) == 1); - CHECK(Package.RemoveAttachment(Level4Hash) == 0); - CHECK(Package.GetAttachments().size() == 0); - - CHECK(PackageCopy != Package); - Package = PackageCopy; - CHECK(PackageCopy == Package); - Package.SetObject(CbObject()); - CHECK(PackageCopy != Package); - CHECK(Package.GetObjectHash() == IoHash()); - } - - // Out of Order - { - CbWriter Writer; - CbAttachment Attachment2(Level2, Level2Hash); - Attachment2.Save(Writer); - CbAttachment Attachment4(Level4); - Attachment4.Save(Writer); - Writer.AddHash(Level1Hash); - Writer.AddObject(Level1); - CbAttachment Attachment3(Level3, Level3Hash); - Attachment3.Save(Writer); - Writer.AddNull(); - - CbFieldIterator Fields = Writer.Save(); - CbPackage FromFields; - FromFields.TryLoad(Fields); - - const CbAttachment* const Level2Attachment = FromFields.FindAttachment(Level2Hash); - REQUIRE(Level2Attachment); - const CbAttachment* const Level3Attachment = FromFields.FindAttachment(Level3Hash); - REQUIRE(Level3Attachment); - const CbAttachment* const Level4Attachment = FromFields.FindAttachment(Level4Hash); - REQUIRE(Level4Attachment); - - CHECK(FromFields.GetObject().Equals(Level1)); - CHECK(FromFields.GetObject().GetOuterBuffer() == Fields.GetOuterBuffer()); - CHECK(FromFields.GetObjectHash() == Level1Hash); - - const MemoryView FieldsOuterBufferView = Fields.GetOuterBuffer().GetView(); - - CHECK(Level2Attachment->AsObject().Equals(Level2)); - CHECK(Level2Attachment->GetHash() == Level2Hash); - - CHECK(Level3Attachment->AsObject().Equals(Level3)); - CHECK(Level3Attachment->GetHash() == Level3Hash); - - CHECK(Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView())); - CHECK(FieldsOuterBufferView.Contains(Level4Attachment->AsBinary().GetView())); - CHECK(Level4Attachment->GetHash() == Level4Hash); - - BinaryWriter WriteStream; - Writer.Save(WriteStream); - CbPackage FromArchive; - BinaryReader ReadAr(MakeMemoryView(WriteStream)); - FromArchive.TryLoad(ReadAr); - - Writer.Reset(); - FromArchive.Save(Writer); - CbFieldIterator Saved = Writer.Save(); - - CHECK(Saved.AsHash() == Level1Hash); - ++Saved; - CHECK(Saved.AsObject().Equals(Level1)); - ++Saved; - CHECK_EQ(Saved.AsObjectAttachment(), Level2Hash); - ++Saved; - CHECK(Saved.AsObject().Equals(Level2)); - ++Saved; - CHECK_EQ(Saved.AsObjectAttachment(), Level3Hash); - ++Saved; - CHECK(Saved.AsObject().Equals(Level3)); - ++Saved; - CHECK_EQ(Saved.AsBinaryAttachment(), Level4Hash); - ++Saved; - SharedBuffer SavedLevel4Buffer = SharedBuffer::MakeView(Saved.AsBinaryView()); - CHECK(SavedLevel4Buffer.GetView().EqualBytes(Level4.GetView())); - ++Saved; - CHECK(Saved.IsNull()); - ++Saved; - CHECK(!Saved); - } - - // Null Attachment - { - CbAttachment NullAttachment; - CbPackage Package; - Package.AddAttachment(NullAttachment); - CHECK(Package.IsNull()); - CHECK_FALSE(bool(Package)); - CHECK(Package.GetAttachments().size() == 0); - CHECK_FALSE(Package.FindAttachment(NullAttachment)); - } - - // Resolve After Merge - { - bool bResolved = false; - CbPackage Package; - Package.AddAttachment(CbAttachment(Level3.GetBuffer())); - Package.AddAttachment(CbAttachment(Level3), [&bResolved](const IoHash& Hash) -> SharedBuffer { - ZEN_UNUSED(Hash); - bResolved = true; - return SharedBuffer(); - }); - CHECK(bResolved); - } -} - -TEST_CASE("usonpackage.invalidpackage") -{ - const auto TestLoad = [](std::initializer_list<uint8_t> RawData, BufferAllocator Allocator = UniqueBuffer::Alloc) { - const MemoryView RawView = MakeMemoryView(RawData); - CbPackage FromArchive; - BinaryReader ReadAr(RawView); - CHECK_FALSE(FromArchive.TryLoad(ReadAr, Allocator)); - }; - const auto AllocFail = [](uint64_t) -> UniqueBuffer { - FAIL_CHECK("Allocation is not expected"); - return UniqueBuffer(); - }; - SUBCASE("Empty") { TestLoad({}, AllocFail); } - SUBCASE("Invalid Initial Field") - { - TestLoad({uint8_t(CbFieldType::None)}); - TestLoad({uint8_t(CbFieldType::Array)}); - TestLoad({uint8_t(CbFieldType::UniformArray)}); - TestLoad({uint8_t(CbFieldType::Binary)}); - TestLoad({uint8_t(CbFieldType::String)}); - TestLoad({uint8_t(CbFieldType::IntegerPositive)}); - TestLoad({uint8_t(CbFieldType::IntegerNegative)}); - TestLoad({uint8_t(CbFieldType::Float32)}); - TestLoad({uint8_t(CbFieldType::Float64)}); - TestLoad({uint8_t(CbFieldType::BoolFalse)}); - TestLoad({uint8_t(CbFieldType::BoolTrue)}); - TestLoad({uint8_t(CbFieldType::ObjectAttachment)}); - TestLoad({uint8_t(CbFieldType::BinaryAttachment)}); - TestLoad({uint8_t(CbFieldType::Uuid)}); - TestLoad({uint8_t(CbFieldType::DateTime)}); - TestLoad({uint8_t(CbFieldType::TimeSpan)}); - TestLoad({uint8_t(CbFieldType::ObjectId)}); - TestLoad({uint8_t(CbFieldType::CustomById)}); - TestLoad({uint8_t(CbFieldType::CustomByName)}); - } - SUBCASE("Size Out Of Bounds") - { - TestLoad({uint8_t(CbFieldType::Object), 1}, AllocFail); - TestLoad({uint8_t(CbFieldType::Object), 0xff, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0}, AllocFail); - } -} - -#endif - -} // namespace zen diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp deleted file mode 100644 index 02148d96a..000000000 --- a/zencore/compactbinaryvalidation.cpp +++ /dev/null @@ -1,664 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/compactbinaryvalidation.h" - -#include <zencore/compactbinarypackage.h> -#include <zencore/endian.h> -#include <zencore/memory.h> -#include <zencore/string.h> -#include <zencore/testing.h> - -#include <algorithm> - -namespace zen { - -namespace CbValidationPrivate { - - template<typename T> - static constexpr inline T ReadUnaligned(const void* const Memory) - { -#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS - return *static_cast<const T*>(Memory); -#else - T Value; - memcpy(&Value, Memory, sizeof(Value)); - return Value; -#endif - } - -} // namespace CbValidationPrivate - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * Adds the given error(s) to the error mask. - * - * This function exists to make validation errors easier to debug by providing one location to set a breakpoint. - */ -ZEN_NOINLINE static void -AddError(CbValidateError& OutError, const CbValidateError InError) -{ - OutError |= InError; -} - -/** - * Validate and read a field type from the view. - * - * A type argument with the HasFieldType flag indicates that the type will not be read from the view. - */ -static CbFieldType -ValidateCbFieldType(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType Type = CbFieldType::HasFieldType) -{ - ZEN_UNUSED(Mode); - if (CbFieldTypeOps::HasFieldType(Type)) - { - if (View.GetSize() >= 1) - { - Type = *static_cast<const CbFieldType*>(View.GetData()); - View += 1; - if (CbFieldTypeOps::HasFieldType(Type)) - { - AddError(Error, CbValidateError::InvalidType); - } - } - else - { - AddError(Error, CbValidateError::OutOfBounds); - View.Reset(); - return CbFieldType::None; - } - } - - if (CbFieldTypeOps::GetSerializedType(Type) != Type) - { - AddError(Error, CbValidateError::InvalidType); - View.Reset(); - } - - return Type; -} - -/** - * Validate and read an unsigned integer from the view. - * - * Modifies the view to start at the end of the value, and adds error flags if applicable. - */ -static uint64_t -ValidateCbUInt(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) -{ - if (View.GetSize() > 0 && View.GetSize() >= MeasureVarUInt(View.GetData())) - { - uint32_t ValueByteCount; - const uint64_t Value = ReadVarUInt(View.GetData(), ValueByteCount); - if (EnumHasAnyFlags(Mode, CbValidateMode::Format) && ValueByteCount > MeasureVarUInt(Value)) - { - AddError(Error, CbValidateError::InvalidInteger); - } - View += ValueByteCount; - return Value; - } - else - { - AddError(Error, CbValidateError::OutOfBounds); - View.Reset(); - return 0; - } -} - -/** - * Validate a 64-bit floating point value from the view. - * - * Modifies the view to start at the end of the value, and adds error flags if applicable. - */ -static void -ValidateCbFloat64(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) -{ - if (View.GetSize() >= sizeof(double)) - { - if (EnumHasAnyFlags(Mode, CbValidateMode::Format)) - { - const uint64_t RawValue = FromNetworkOrder(CbValidationPrivate::ReadUnaligned<uint64_t>(View.GetData())); - const double Value = reinterpret_cast<const double&>(RawValue); - if (Value == double(float(Value))) - { - AddError(Error, CbValidateError::InvalidFloat); - } - } - View += sizeof(double); - } - else - { - AddError(Error, CbValidateError::OutOfBounds); - View.Reset(); - } -} - -/** - * Validate and read a string from the view. - * - * Modifies the view to start at the end of the string, and adds error flags if applicable. - */ -static std::string_view -ValidateCbString(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) -{ - const uint64_t NameSize = ValidateCbUInt(View, Mode, Error); - if (View.GetSize() >= NameSize) - { - const std::string_view Name(static_cast<const char*>(View.GetData()), static_cast<int32_t>(NameSize)); - View += NameSize; - return Name; - } - else - { - AddError(Error, CbValidateError::OutOfBounds); - View.Reset(); - return std::string_view(); - } -} - -static CbFieldView ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ExternalType); - -/** A type that checks whether all validated fields are of the same type. */ -class CbUniformFieldsValidator -{ -public: - inline explicit CbUniformFieldsValidator(CbFieldType InExternalType) : ExternalType(InExternalType) {} - - inline CbFieldView ValidateField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) - { - const void* const FieldData = View.GetData(); - if (CbFieldView Field = ValidateCbField(View, Mode, Error, ExternalType)) - { - ++FieldCount; - if (CbFieldTypeOps::HasFieldType(ExternalType)) - { - const CbFieldType FieldType = *static_cast<const CbFieldType*>(FieldData); - if (FieldCount == 1) - { - FirstType = FieldType; - } - else if (FieldType != FirstType) - { - bUniform = false; - } - } - return Field; - } - - // It may not safe to check for uniformity if the field was invalid. - bUniform = false; - return CbFieldView(); - } - - inline bool IsUniform() const { return FieldCount > 0 && bUniform; } - -private: - uint32_t FieldCount = 0; - bool bUniform = true; - CbFieldType FirstType = CbFieldType::None; - CbFieldType ExternalType; -}; - -static void -ValidateCbObject(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ObjectType) -{ - const uint64_t Size = ValidateCbUInt(View, Mode, Error); - MemoryView ObjectView = View.Left(Size); - View += Size; - - if (Size > 0) - { - std::vector<std::string_view> Names; - - const bool bUniformObject = CbFieldTypeOps::GetType(ObjectType) == CbFieldType::UniformObject; - const CbFieldType ExternalType = bUniformObject ? ValidateCbFieldType(ObjectView, Mode, Error) : CbFieldType::HasFieldType; - CbUniformFieldsValidator UniformValidator(ExternalType); - do - { - if (CbFieldView Field = UniformValidator.ValidateField(ObjectView, Mode, Error)) - { - if (EnumHasAnyFlags(Mode, CbValidateMode::Names)) - { - if (Field.HasName()) - { - Names.push_back(Field.GetName()); - } - else - { - AddError(Error, CbValidateError::MissingName); - } - } - } - } while (!ObjectView.IsEmpty()); - - if (EnumHasAnyFlags(Mode, CbValidateMode::Names) && Names.size() > 1) - { - std::sort(begin(Names), end(Names), [](std::string_view L, std::string_view R) { return L.compare(R) < 0; }); - - for (const std::string_view *NamesIt = Names.data(), *NamesEnd = NamesIt + Names.size() - 1; NamesIt != NamesEnd; ++NamesIt) - { - if (NamesIt[0] == NamesIt[1]) - { - AddError(Error, CbValidateError::DuplicateName); - break; - } - } - } - - if (!bUniformObject && EnumHasAnyFlags(Mode, CbValidateMode::Format) && UniformValidator.IsUniform()) - { - AddError(Error, CbValidateError::NonUniformObject); - } - } -} - -static void -ValidateCbArray(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ArrayType) -{ - const uint64_t Size = ValidateCbUInt(View, Mode, Error); - MemoryView ArrayView = View.Left(Size); - View += Size; - - const uint64_t Count = ValidateCbUInt(ArrayView, Mode, Error); - const uint64_t FieldsSize = ArrayView.GetSize(); - const bool bUniformArray = CbFieldTypeOps::GetType(ArrayType) == CbFieldType::UniformArray; - const CbFieldType ExternalType = bUniformArray ? ValidateCbFieldType(ArrayView, Mode, Error) : CbFieldType::HasFieldType; - CbUniformFieldsValidator UniformValidator(ExternalType); - - for (uint64_t Index = 0; Index < Count; ++Index) - { - if (CbFieldView Field = UniformValidator.ValidateField(ArrayView, Mode, Error)) - { - if (Field.HasName() && EnumHasAnyFlags(Mode, CbValidateMode::Names)) - { - AddError(Error, CbValidateError::ArrayName); - } - } - } - - if (!bUniformArray && EnumHasAnyFlags(Mode, CbValidateMode::Format) && UniformValidator.IsUniform() && FieldsSize > Count) - { - AddError(Error, CbValidateError::NonUniformArray); - } -} - -static CbFieldView -ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, const CbFieldType ExternalType = CbFieldType::HasFieldType) -{ - const MemoryView FieldView = View; - const CbFieldType Type = ValidateCbFieldType(View, Mode, Error, ExternalType); - [[maybe_unused]] const std::string_view Name = - CbFieldTypeOps::HasFieldName(Type) ? ValidateCbString(View, Mode, Error) : std::string_view(); - - auto ValidateFixedPayload = [&View, &Error](uint32_t PayloadSize) { - if (View.GetSize() >= PayloadSize) - { - View += PayloadSize; - } - else - { - AddError(Error, CbValidateError::OutOfBounds); - View.Reset(); - } - }; - - if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds | CbValidateError::InvalidType)) - { - return CbFieldView(); - } - - switch (CbFieldType FieldType = CbFieldTypeOps::GetType(Type)) - { - default: - case CbFieldType::None: - AddError(Error, CbValidateError::InvalidType); - View.Reset(); - break; - case CbFieldType::Null: - case CbFieldType::BoolFalse: - case CbFieldType::BoolTrue: - if (FieldView == View) - { - // Reset the view because a zero-sized field can cause infinite field iteration. - AddError(Error, CbValidateError::InvalidType); - View.Reset(); - } - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - ValidateCbObject(View, Mode, Error, FieldType); - break; - case CbFieldType::Array: - case CbFieldType::UniformArray: - ValidateCbArray(View, Mode, Error, FieldType); - break; - case CbFieldType::Binary: - { - const uint64_t ValueSize = ValidateCbUInt(View, Mode, Error); - if (View.GetSize() < ValueSize) - { - AddError(Error, CbValidateError::OutOfBounds); - View.Reset(); - } - else - { - View += ValueSize; - } - break; - } - case CbFieldType::String: - ValidateCbString(View, Mode, Error); - break; - case CbFieldType::IntegerPositive: - ValidateCbUInt(View, Mode, Error); - break; - case CbFieldType::IntegerNegative: - ValidateCbUInt(View, Mode, Error); - break; - case CbFieldType::Float32: - ValidateFixedPayload(4); - break; - case CbFieldType::Float64: - ValidateCbFloat64(View, Mode, Error); - break; - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - case CbFieldType::Hash: - ValidateFixedPayload(20); - break; - case CbFieldType::Uuid: - ValidateFixedPayload(16); - break; - case CbFieldType::DateTime: - case CbFieldType::TimeSpan: - ValidateFixedPayload(8); - break; - case CbFieldType::ObjectId: - ValidateFixedPayload(12); - break; - case CbFieldType::CustomById: - case CbFieldType::CustomByName: - ZEN_NOT_IMPLEMENTED(); // TODO: FIX! - break; - } - - if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds | CbValidateError::InvalidType)) - { - return CbFieldView(); - } - - return CbFieldView(FieldView.GetData(), ExternalType); -} - -static CbFieldView -ValidateCbPackageField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) -{ - if (View.IsEmpty()) - { - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - return CbFieldView(); - } - if (CbFieldView Field = ValidateCbField(View, Mode, Error)) - { - if (Field.HasName() && EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - return Field; - } - return CbFieldView(); -} - -static IoHash -ValidateCbPackageAttachment(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error) -{ - if (const CbObjectView ObjectView = Value.AsObjectView(); !Value.HasError()) - { - return ObjectView.GetHash(); - } - - if (const IoHash ObjectAttachmentHash = Value.AsObjectAttachment(); !Value.HasError()) - { - if (CbFieldView ObjectField = ValidateCbPackageField(View, Mode, Error)) - { - const CbObjectView InnerObjectView = ObjectField.AsObjectView(); - if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && ObjectField.HasError()) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - else if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (ObjectAttachmentHash != InnerObjectView.GetHash())) - { - AddError(Error, CbValidateError::InvalidPackageHash); - } - return ObjectAttachmentHash; - } - } - else if (const IoHash BinaryAttachmentHash = Value.AsBinaryAttachment(); !Value.HasError()) - { - if (CbFieldView BinaryField = ValidateCbPackageField(View, Mode, Error)) - { - const MemoryView BinaryView = BinaryField.AsBinaryView(); - if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && BinaryField.HasError()) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - else - { - if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && BinaryView.IsEmpty()) - { - AddError(Error, CbValidateError::NullPackageAttachment); - } - if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (BinaryAttachmentHash != IoHash::HashBuffer(BinaryView))) - { - AddError(Error, CbValidateError::InvalidPackageHash); - } - } - return BinaryAttachmentHash; - } - } - else if (const MemoryView BinaryView = Value.AsBinaryView(); !Value.HasError()) - { - if (BinaryView.GetSize() > 0) - { - IoHash DecodedHash; - uint64_t DecodedRawSize; - CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView), DecodedHash, DecodedRawSize); - if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && Buffer.IsNull()) - { - AddError(Error, CbValidateError::NullPackageAttachment); - } - if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (DecodedHash != IoHash::HashBuffer(Buffer.DecompressToComposite()))) - { - AddError(Error, CbValidateError::InvalidPackageHash); - } - return DecodedHash; - } - else - { - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - AddError(Error, CbValidateError::NullPackageAttachment); - } - return IoHash::HashBuffer(MemoryView()); - } - } - else - { - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - } - - return IoHash(); -} - -static IoHash -ValidateCbPackageObject(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error) -{ - if (IoHash RootObjectHash = Value.AsHash(); !Value.HasError() && !Value.IsAttachment()) - { - CbFieldView RootObjectField = ValidateCbPackageField(View, Mode, Error); - - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - if (RootObjectField.HasError()) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - } - - const CbObjectView RootObjectView = RootObjectField.AsObjectView(); - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - if (!RootObjectView) - { - AddError(Error, CbValidateError::NullPackageObject); - } - } - - if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (RootObjectHash != RootObjectView.GetHash())) - { - AddError(Error, CbValidateError::InvalidPackageHash); - } - - return RootObjectHash; - } - else - { - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - } - - return IoHash(); -} - -CbValidateError -ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type) -{ - CbValidateError Error = CbValidateError::None; - if (EnumHasAnyFlags(Mode, CbValidateMode::All)) - { - ValidateCbField(View, Mode, Error, Type); - if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding)) - { - AddError(Error, CbValidateError::Padding); - } - } - return Error; -} - -CbValidateError -ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode) -{ - CbValidateError Error = CbValidateError::None; - if (EnumHasAnyFlags(Mode, CbValidateMode::All)) - { - while (!View.IsEmpty()) - { - ValidateCbField(View, Mode, Error); - } - } - return Error; -} - -CbValidateError -ValidateObjectAttachment(MemoryView View, CbValidateMode Mode) -{ - CbValidateError Error = CbValidateError::None; - if (EnumHasAnyFlags(Mode, CbValidateMode::All)) - { - if (CbFieldView Value = ValidateCbPackageField(View, Mode, Error)) - { - ValidateCbPackageAttachment(Value, View, Mode, Error); - } - if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding)) - { - AddError(Error, CbValidateError::Padding); - } - } - return Error; -} - -CbValidateError -ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode) -{ - std::vector<IoHash> Attachments; - CbValidateError Error = CbValidateError::None; - if (EnumHasAnyFlags(Mode, CbValidateMode::All)) - { - uint32_t ObjectCount = 0; - while (CbFieldView Value = ValidateCbPackageField(View, Mode, Error)) - { - if (Value.IsHash() && !Value.IsAttachment()) - { - ValidateCbPackageObject(Value, View, Mode, Error); - if (++ObjectCount > 1 && EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - AddError(Error, CbValidateError::MultiplePackageObjects); - } - } - else if (Value.IsBinary() || Value.IsAttachment() || Value.IsObject()) - { - const IoHash Hash = ValidateCbPackageAttachment(Value, View, Mode, Error); - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - Attachments.push_back(Hash); - } - } - else if (Value.IsNull()) - { - break; - } - else if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - - if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds)) - { - break; - } - } - - if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding)) - { - AddError(Error, CbValidateError::Padding); - } - - if (Attachments.size() && EnumHasAnyFlags(Mode, CbValidateMode::Package)) - { - std::sort(begin(Attachments), end(Attachments)); - for (const IoHash *It = Attachments.data(), *End = It + Attachments.size() - 1; It != End; ++It) - { - if (It[0] == It[1]) - { - AddError(Error, CbValidateError::DuplicateAttachments); - break; - } - } - } - } - return Error; -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#if ZEN_WITH_TESTS -void -usonvalidation_forcelink() -{ -} - -TEST_CASE("usonvalidation") -{ - SUBCASE("Basic") {} -} -#endif - -} // namespace zen diff --git a/zencore/compositebuffer.cpp b/zencore/compositebuffer.cpp deleted file mode 100644 index 735020451..000000000 --- a/zencore/compositebuffer.cpp +++ /dev/null @@ -1,446 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/compositebuffer.h> - -#include <zencore/sharedbuffer.h> -#include <zencore/testing.h> - -namespace zen { - -const CompositeBuffer CompositeBuffer::Null; - -void -CompositeBuffer::Reset() -{ - m_Segments.clear(); -} - -uint64_t -CompositeBuffer::GetSize() const -{ - uint64_t Accum = 0; - - for (const SharedBuffer& It : m_Segments) - { - Accum += It.GetSize(); - } - - return Accum; -} - -bool -CompositeBuffer::IsOwned() const -{ - for (const SharedBuffer& It : m_Segments) - { - if (It.IsOwned() == false) - { - return false; - } - } - return true; -} - -CompositeBuffer -CompositeBuffer::MakeOwned() const& -{ - return CompositeBuffer(*this).MakeOwned(); -} - -CompositeBuffer -CompositeBuffer::MakeOwned() && -{ - for (SharedBuffer& Segment : m_Segments) - { - Segment = std::move(Segment).MakeOwned(); - } - return std::move(*this); -} - -SharedBuffer -CompositeBuffer::Flatten() const& -{ - switch (m_Segments.size()) - { - case 0: - return SharedBuffer(); - case 1: - return m_Segments[0]; - default: - UniqueBuffer Buffer = UniqueBuffer::Alloc(GetSize()); - MutableMemoryView OutView = Buffer.GetMutableView(); - - for (const SharedBuffer& Segment : m_Segments) - { - OutView.CopyFrom(Segment.GetView()); - OutView += Segment.GetSize(); - } - - return Buffer.MoveToShared(); - } -} - -SharedBuffer -CompositeBuffer::Flatten() && -{ - return m_Segments.size() == 1 ? std::move(m_Segments[0]) : std::as_const(*this).Flatten(); -} - -CompositeBuffer -CompositeBuffer::Mid(uint64_t Offset, uint64_t Size) const -{ - const uint64_t BufferSize = GetSize(); - Offset = Min(Offset, BufferSize); - Size = Min(Size, BufferSize - Offset); - CompositeBuffer Buffer; - IterateRange(Offset, Size, [&Buffer](MemoryView View, const SharedBuffer& ViewOuter) { - Buffer.m_Segments.push_back(SharedBuffer::MakeView(View, ViewOuter)); - }); - return Buffer; -} - -MemoryView -CompositeBuffer::ViewOrCopyRange(uint64_t Offset, uint64_t Size, UniqueBuffer& CopyBuffer) const -{ - MemoryView View; - IterateRange(Offset, Size, [Size, &View, &CopyBuffer, WriteView = MutableMemoryView()](MemoryView Segment) mutable { - if (Size == Segment.GetSize()) - { - View = Segment; - } - else - { - if (WriteView.IsEmpty()) - { - if (CopyBuffer.GetSize() < Size) - { - CopyBuffer = UniqueBuffer::Alloc(Size); - } - View = WriteView = CopyBuffer.GetMutableView().Left(Size); - } - WriteView = WriteView.CopyFrom(Segment); - } - }); - return View; -} - -CompositeBuffer::Iterator -CompositeBuffer::GetIterator(uint64_t Offset) const -{ - size_t SegmentCount = m_Segments.size(); - size_t SegmentIndex = 0; - while (SegmentIndex < SegmentCount) - { - size_t SegmentSize = m_Segments[SegmentIndex].GetSize(); - if (Offset < SegmentSize) - { - return {.SegmentIndex = SegmentIndex, .OffsetInSegment = Offset}; - } - Offset -= SegmentSize; - SegmentIndex++; - } - return {.SegmentIndex = ~0ull, .OffsetInSegment = ~0ull}; -} - -MemoryView -CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const -{ - // We use a sub range IoBuffer when we want to copy data from a segment. - // This means we will only materialize that range of the segment when doing - // GetView() rather than the full segment. - // A hot path for this code is when we call CompressedBuffer::FromCompressed which - // is only interested in reading the header (first 64 bytes or so) and then throws - // away the materialized data. - MutableMemoryView WriteView; - size_t SegmentCount = m_Segments.size(); - ZEN_ASSERT(It.SegmentIndex < SegmentCount); - uint64_t SizeLeft = Size; - while (SizeLeft > 0 && It.SegmentIndex < SegmentCount) - { - const SharedBuffer& Segment = m_Segments[It.SegmentIndex]; - size_t SegmentSize = Segment.GetSize(); - if (Size == SizeLeft && Size <= (SegmentSize - It.OffsetInSegment)) - { - IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, SizeLeft); - MemoryView View = SubSegment.GetView(); - It.OffsetInSegment += SizeLeft; - ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); - if (It.OffsetInSegment == SegmentSize) - { - It.SegmentIndex++; - It.OffsetInSegment = 0; - } - return View; - } - if (WriteView.GetSize() == 0) - { - if (CopyBuffer.GetSize() < Size) - { - CopyBuffer = UniqueBuffer::Alloc(Size); - } - WriteView = CopyBuffer.GetMutableView(); - } - size_t CopySize = zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft); - IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, CopySize); - MemoryView ReadView = SubSegment.GetView(); - WriteView = WriteView.CopyFrom(ReadView); - It.OffsetInSegment += CopySize; - ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); - if (It.OffsetInSegment == SegmentSize) - { - It.SegmentIndex++; - It.OffsetInSegment = 0; - } - SizeLeft -= CopySize; - } - return CopyBuffer.GetView().Left(Size - SizeLeft); -} - -void -CompositeBuffer::CopyTo(MutableMemoryView WriteView, Iterator& It) const -{ - // We use a sub range IoBuffer when we want to copy data from a segment. - // This means we will only materialize that range of the segment when doing - // GetView() rather than the full segment. - // A hot path for this code is when we call CompressedBuffer::FromCompressed which - // is only interested in reading the header (first 64 bytes or so) and then throws - // away the materialized data. - - size_t SizeLeft = WriteView.GetSize(); - size_t SegmentCount = m_Segments.size(); - ZEN_ASSERT(It.SegmentIndex < SegmentCount); - while (WriteView.GetSize() > 0 && It.SegmentIndex < SegmentCount) - { - const SharedBuffer& Segment = m_Segments[It.SegmentIndex]; - size_t SegmentSize = Segment.GetSize(); - size_t CopySize = zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft); - IoBuffer SubSegment(Segment.AsIoBuffer(), It.OffsetInSegment, CopySize); - MemoryView ReadView = SubSegment.GetView(); - WriteView = WriteView.CopyFrom(ReadView); - It.OffsetInSegment += CopySize; - ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize); - if (It.OffsetInSegment == SegmentSize) - { - It.SegmentIndex++; - It.OffsetInSegment = 0; - } - SizeLeft -= CopySize; - } -} - -void -CompositeBuffer::CopyTo(MutableMemoryView Target, uint64_t Offset) const -{ - IterateRange(Offset, Target.GetSize(), [Target](MemoryView View, [[maybe_unused]] const SharedBuffer& ViewOuter) mutable { - Target = Target.CopyFrom(View); - }); -} - -void -CompositeBuffer::IterateRange(uint64_t Offset, uint64_t Size, std::function<void(MemoryView View)> Visitor) const -{ - IterateRange(Offset, Size, [Visitor](MemoryView View, [[maybe_unused]] const SharedBuffer& ViewOuter) { Visitor(View); }); -} - -void -CompositeBuffer::IterateRange(uint64_t Offset, - uint64_t Size, - std::function<void(MemoryView View, const SharedBuffer& ViewOuter)> Visitor) const -{ - ZEN_ASSERT(Offset + Size <= GetSize()); - for (const SharedBuffer& Segment : m_Segments) - { - if (const uint64_t SegmentSize = Segment.GetSize(); Offset <= SegmentSize) - { - const MemoryView View = Segment.GetView().Mid(Offset, Size); - Offset = 0; - if (Size == 0 || !View.IsEmpty()) - { - Visitor(View, Segment); - } - Size -= View.GetSize(); - if (Size == 0) - { - break; - } - } - else - { - Offset -= SegmentSize; - } - } -} - -#if ZEN_WITH_TESTS -TEST_CASE("CompositeBuffer Null") -{ - CompositeBuffer Buffer; - CHECK(Buffer.IsNull()); - CHECK(Buffer.IsOwned()); - CHECK(Buffer.MakeOwned().IsNull()); - CHECK(Buffer.Flatten().IsNull()); - CHECK(Buffer.Mid(0, 0).IsNull()); - CHECK(Buffer.GetSize() == 0); - CHECK(Buffer.GetSegments().size() == 0); - - UniqueBuffer CopyBuffer; - CHECK(Buffer.ViewOrCopyRange(0, 0, CopyBuffer).IsEmpty()); - CHECK(CopyBuffer.IsNull()); - - MutableMemoryView CopyView; - Buffer.CopyTo(CopyView); - - uint32_t VisitCount = 0; - Buffer.IterateRange(0, 0, [&VisitCount](MemoryView) { ++VisitCount; }); - CHECK(VisitCount == 0); -} - -TEST_CASE("CompositeBuffer Empty") -{ - const uint8_t EmptyArray[]{0}; - const SharedBuffer EmptyView = SharedBuffer::MakeView(EmptyArray, 0); - CompositeBuffer Buffer(EmptyView); - CHECK(Buffer.IsNull() == false); - CHECK(Buffer.IsOwned() == false); - CHECK(Buffer.MakeOwned().IsNull() == false); - CHECK(Buffer.MakeOwned().IsOwned() == true); - CHECK(Buffer.Flatten() == EmptyView); - CHECK(Buffer.Mid(0, 0).Flatten() == EmptyView); - CHECK(Buffer.GetSize() == 0); - CHECK(Buffer.GetSegments().size() == 1); - CHECK(Buffer.GetSegments()[0] == EmptyView); - - UniqueBuffer CopyBuffer; - CHECK(Buffer.ViewOrCopyRange(0, 0, CopyBuffer) == EmptyView.GetView()); - CHECK(CopyBuffer.IsNull()); - - MutableMemoryView CopyView; - Buffer.CopyTo(CopyView); - - uint32_t VisitCount = 0; - Buffer.IterateRange(0, 0, [&VisitCount](MemoryView) { ++VisitCount; }); - CHECK(VisitCount == 1); -} - -TEST_CASE("CompositeBuffer Empty[1]") -{ - const uint8_t EmptyArray[1]{}; - const SharedBuffer EmptyView1 = SharedBuffer::MakeView(EmptyArray, 0); - const SharedBuffer EmptyView2 = SharedBuffer::MakeView(EmptyArray + 1, 0); - CompositeBuffer Buffer(EmptyView1, EmptyView2); - CHECK(Buffer.Mid(0, 0).Flatten() == EmptyView1); - CHECK(Buffer.GetSize() == 0); - CHECK(Buffer.GetSegments().size() == 2); - CHECK(Buffer.GetSegments()[0] == EmptyView1); - CHECK(Buffer.GetSegments()[1] == EmptyView2); - - UniqueBuffer CopyBuffer; - CHECK(Buffer.ViewOrCopyRange(0, 0, CopyBuffer) == EmptyView1.GetView()); - CHECK(CopyBuffer.IsNull()); - - MutableMemoryView CopyView; - Buffer.CopyTo(CopyView); - - uint32_t VisitCount = 0; - Buffer.IterateRange(0, 0, [&VisitCount](MemoryView) { ++VisitCount; }); - CHECK(VisitCount == 1); -} - -TEST_CASE("CompositeBuffer Flat") -{ - const uint8_t FlatArray[]{1, 2, 3, 4, 5, 6, 7, 8}; - const SharedBuffer FlatView = SharedBuffer::Clone(MakeMemoryView(FlatArray)); - CompositeBuffer Buffer(FlatView); - - CHECK(Buffer.IsNull() == false); - CHECK(Buffer.IsOwned() == true); - CHECK(Buffer.Flatten() == FlatView); - CHECK(Buffer.MakeOwned().Flatten() == FlatView); - CHECK(Buffer.Mid(0).Flatten() == FlatView); - CHECK(Buffer.Mid(4).Flatten().GetView() == FlatView.GetView().Mid(4)); - CHECK(Buffer.Mid(8).Flatten().GetView() == FlatView.GetView().Mid(8)); - CHECK(Buffer.Mid(4, 2).Flatten().GetView() == FlatView.GetView().Mid(4, 2)); - CHECK(Buffer.Mid(8, 0).Flatten().GetView() == FlatView.GetView().Mid(8, 0)); - CHECK(Buffer.GetSize() == sizeof(FlatArray)); - CHECK(Buffer.GetSegments().size() == 1); - CHECK(Buffer.GetSegments()[0] == FlatView); - - UniqueBuffer CopyBuffer; - CHECK(Buffer.ViewOrCopyRange(0, sizeof(FlatArray), CopyBuffer) == FlatView.GetView()); - CHECK(CopyBuffer.IsNull()); - - uint8_t CopyArray[sizeof(FlatArray) - 3]; - Buffer.CopyTo(MakeMutableMemoryView(CopyArray), 3); - CHECK(MakeMemoryView(CopyArray).EqualBytes(MakeMemoryView(FlatArray) + 3)); - - uint32_t VisitCount = 0; - Buffer.IterateRange(0, sizeof(FlatArray), [&VisitCount](MemoryView) { ++VisitCount; }); - CHECK(VisitCount == 1); -} - -TEST_CASE("CompositeBuffer Composite") -{ - const uint8_t FlatArray[]{1, 2, 3, 4, 5, 6, 7, 8}; - const SharedBuffer FlatView1 = SharedBuffer::MakeView(MakeMemoryView(FlatArray).Left(4)); - const SharedBuffer FlatView2 = SharedBuffer::MakeView(MakeMemoryView(FlatArray).Right(4)); - CompositeBuffer Buffer(FlatView1, FlatView2); - - CHECK(Buffer.IsNull() == false); - CHECK(Buffer.IsOwned() == false); - CHECK(Buffer.Flatten().GetView().EqualBytes(MakeMemoryView(FlatArray))); - CHECK(Buffer.Mid(2, 4).Flatten().GetView().EqualBytes(MakeMemoryView(FlatArray).Mid(2, 4))); - CHECK(Buffer.Mid(0, 4).Flatten() == FlatView1); - CHECK(Buffer.Mid(4, 4).Flatten() == FlatView2); - CHECK(Buffer.GetSize() == sizeof(FlatArray)); - CHECK(Buffer.GetSegments().size() == 2); - CHECK(Buffer.GetSegments()[0] == FlatView1); - CHECK(Buffer.GetSegments()[1] == FlatView2); - - UniqueBuffer CopyBuffer; - - CHECK(Buffer.ViewOrCopyRange(0, 4, CopyBuffer) == FlatView1.GetView()); - CHECK(CopyBuffer.IsNull() == true); - CHECK(Buffer.ViewOrCopyRange(4, 4, CopyBuffer) == FlatView2.GetView()); - CHECK(CopyBuffer.IsNull() == true); - CHECK(Buffer.ViewOrCopyRange(3, 2, CopyBuffer).EqualBytes(MakeMemoryView(FlatArray).Mid(3, 2))); - CHECK(CopyBuffer.GetSize() == 2); - CHECK(Buffer.ViewOrCopyRange(1, 6, CopyBuffer).EqualBytes(MakeMemoryView(FlatArray).Mid(1, 6))); - CHECK(CopyBuffer.GetSize() == 6); - CHECK(Buffer.ViewOrCopyRange(2, 4, CopyBuffer).EqualBytes(MakeMemoryView(FlatArray).Mid(2, 4))); - CHECK(CopyBuffer.GetSize() == 6); - - uint8_t CopyArray[4]; - Buffer.CopyTo(MakeMutableMemoryView(CopyArray), 2); - CHECK(MakeMemoryView(CopyArray).EqualBytes(MakeMemoryView(FlatArray).Mid(2, 4))); - - uint32_t VisitCount = 0; - Buffer.IterateRange(0, sizeof(FlatArray), [&VisitCount](MemoryView) { ++VisitCount; }); - CHECK(VisitCount == 2); - - const auto TestIterateRange = - [&Buffer](uint64_t Offset, uint64_t Size, MemoryView ExpectedView, const SharedBuffer& ExpectedViewOuter) { - uint32_t VisitCount = 0; - MemoryView ActualView; - SharedBuffer ActualViewOuter; - Buffer.IterateRange(Offset, Size, [&VisitCount, &ActualView, &ActualViewOuter](MemoryView View, const SharedBuffer& ViewOuter) { - ++VisitCount; - ActualView = View; - ActualViewOuter = ViewOuter; - }); - CHECK(VisitCount == 1); - CHECK(ActualView == ExpectedView); - CHECK(ActualViewOuter == ExpectedViewOuter); - }; - TestIterateRange(0, 4, MakeMemoryView(FlatArray).Mid(0, 4), FlatView1); - TestIterateRange(4, 0, MakeMemoryView(FlatArray).Mid(4, 0), FlatView1); - TestIterateRange(4, 4, MakeMemoryView(FlatArray).Mid(4, 4), FlatView2); - TestIterateRange(8, 0, MakeMemoryView(FlatArray).Mid(8, 0), FlatView2); -} - -void -compositebuffer_forcelink() -{ -} -#endif - -} // namespace zen diff --git a/zencore/compress.cpp b/zencore/compress.cpp deleted file mode 100644 index e89f02d68..000000000 --- a/zencore/compress.cpp +++ /dev/null @@ -1,1353 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/compress.h> - -#include <zencore/blake3.h> -#include <zencore/compositebuffer.h> -#include <zencore/crc32.h> -#include <zencore/endian.h> -#include <zencore/iohash.h> -#include <zencore/testing.h> - -#include "../thirdparty/Oodle/include/oodle2.h" -#if ZEN_PLATFORM_WINDOWS -# pragma comment(lib, "oo2core_win64.lib") -#endif - -#include <lz4.h> -#include <functional> -#include <limits> - -namespace zen::detail { - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -static constexpr uint64_t DefaultBlockSize = 256 * 1024; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** Method used to compress the data in a compressed buffer. */ -enum class CompressionMethod : uint8_t -{ - /** Header is followed by one uncompressed block. */ - None = 0, - /** Header is followed by an array of compressed block sizes then the compressed blocks. */ - Oodle = 3, - /** Header is followed by an array of compressed block sizes then the compressed blocks. */ - LZ4 = 4, -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** Header used on every compressed buffer. Always stored in big-endian format. */ -struct BufferHeader -{ - static constexpr uint32_t ExpectedMagic = 0xb7756362; // <dot>ucb - - uint32_t Magic = ExpectedMagic; // A magic number to identify a compressed buffer. Always 0xb7756362. - uint32_t Crc32 = 0; // A CRC-32 used to check integrity of the buffer. Uses the polynomial 0x04c11db7 - CompressionMethod Method = - CompressionMethod::None; // The method used to compress the buffer. Affects layout of data following the header - uint8_t Compressor = 0; // The method-specific compressor used to compress the buffer. - uint8_t CompressionLevel = 0; // The method-specific compression level used to compress the buffer. - uint8_t BlockSizeExponent = 0; // The power of two size of every uncompressed block except the last. Size is 1 << BlockSizeExponent - uint32_t BlockCount = 0; // The number of blocks that follow the header - uint64_t TotalRawSize = 0; // The total size of the uncompressed data - uint64_t TotalCompressedSize = 0; // The total size of the compressed data including the header - BLAKE3 RawHash; // The hash of the uncompressed data - - /** Checks validity of the buffer based on the magic number, method, and CRC-32. */ - static bool IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize); - static bool IsValid(const SharedBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) - { - return IsValid(CompositeBuffer(CompressedData), OutRawHash, OutRawSize); - } - - /** Read a header from a buffer that is at least sizeof(BufferHeader) without any validation. */ - static BufferHeader Read(const CompositeBuffer& CompressedData) - { - BufferHeader Header; - if (sizeof(BufferHeader) <= CompressedData.GetSize()) - { - // if (CompressedData.GetSegments()[0].AsIoBuffer().IsWholeFile()) - // { - // ZEN_ASSERT(true); - // } - CompositeBuffer::Iterator It; - CompressedData.CopyTo(MakeMutableMemoryView(&Header, &Header + 1), It); - Header.ByteSwap(); - } - return Header; - } - - /** - * Write a header to a memory view that is at least sizeof(BufferHeader). - * - * @param HeaderView View of the header to write, including any method-specific header data. - */ - void Write(MutableMemoryView HeaderView) const - { - BufferHeader Header = *this; - Header.ByteSwap(); - HeaderView.CopyFrom(MakeMemoryView(&Header, &Header + 1)); - Header.ByteSwap(); - Header.Crc32 = CalculateCrc32(HeaderView); - Header.ByteSwap(); - HeaderView.CopyFrom(MakeMemoryView(&Header, &Header + 1)); - } - - void ByteSwap() - { - Magic = zen::ByteSwap(Magic); - Crc32 = zen::ByteSwap(Crc32); - BlockCount = zen::ByteSwap(BlockCount); - TotalRawSize = zen::ByteSwap(TotalRawSize); - TotalCompressedSize = zen::ByteSwap(TotalCompressedSize); - } - - /** Calculate the CRC-32 from a view of a header including any method-specific header data. */ - static uint32_t CalculateCrc32(MemoryView HeaderView) - { - uint32_t Crc32 = 0; - constexpr uint64_t MethodOffset = offsetof(BufferHeader, Method); - for (MemoryView View = HeaderView + MethodOffset; const uint64_t ViewSize = View.GetSize();) - { - const int32_t Size = static_cast<int32_t>(zen::Min<uint64_t>(ViewSize, /* INT_MAX */ 2147483647u)); - Crc32 = zen::MemCrc32(View.GetData(), Size, Crc32); - View += Size; - } - return Crc32; - } -}; - -static_assert(sizeof(BufferHeader) == 64, "BufferHeader is the wrong size."); - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class BaseEncoder -{ -public: - virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0; -}; - -class BaseDecoder -{ -public: - virtual CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const = 0; - virtual bool TryDecompressTo(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - MutableMemoryView RawView, - uint64_t RawOffset) const = 0; - virtual uint64_t GetHeaderSize(const BufferHeader& Header) const = 0; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class NoneEncoder final : public BaseEncoder -{ -public: - [[nodiscard]] CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t /* BlockSize */) const final - { - BufferHeader Header; - Header.Method = CompressionMethod::None; - Header.BlockCount = 1; - Header.TotalRawSize = RawData.GetSize(); - Header.TotalCompressedSize = Header.TotalRawSize + sizeof(BufferHeader); - Header.RawHash = BLAKE3::HashBuffer(RawData); - - UniqueBuffer HeaderData = UniqueBuffer::Alloc(sizeof(BufferHeader)); - Header.Write(HeaderData); - return CompositeBuffer(HeaderData.MoveToShared(), RawData.MakeOwned()); - } -}; - -class NoneDecoder final : public BaseDecoder -{ -public: - [[nodiscard]] CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const final - { - if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() && - Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader)) - { - return CompressedData.Mid(sizeof(BufferHeader), Header.TotalRawSize).MakeOwned(); - } - return CompositeBuffer(); - } - - [[nodiscard]] bool TryDecompressTo(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - MutableMemoryView RawView, - uint64_t RawOffset) const final - { - if (Header.Method == CompressionMethod::None && RawOffset + RawView.GetSize() <= Header.TotalRawSize && - Header.TotalCompressedSize == CompressedData.GetSize() && - Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader)) - { - CompressedData.CopyTo(RawView, sizeof(BufferHeader) + RawOffset); - return true; - } - return false; - } - - [[nodiscard]] uint64_t GetHeaderSize(const BufferHeader&) const final { return sizeof(BufferHeader); } -}; - -////////////////////////////////////////////////////////////////////////// - -class BlockEncoder : public BaseEncoder -{ -public: - CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const final; - -protected: - virtual CompressionMethod GetMethod() const = 0; - virtual uint8_t GetCompressor() const = 0; - virtual uint8_t GetCompressionLevel() const = 0; - virtual uint64_t CompressBlockBound(uint64_t RawSize) const = 0; - virtual bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const = 0; - -private: - uint64_t GetCompressedBlocksBound(uint64_t BlockCount, uint64_t BlockSize, uint64_t RawSize) const - { - switch (BlockCount) - { - case 0: - return 0; - case 1: - return CompressBlockBound(RawSize); - default: - return CompressBlockBound(BlockSize) - BlockSize + RawSize; - } - } -}; - -CompositeBuffer -BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) const -{ - ZEN_ASSERT(IsPow2(BlockSize) && (BlockSize <= (1u << 31))); - - const uint64_t RawSize = RawData.GetSize(); - BLAKE3Stream RawHash; - - const uint64_t BlockCount = RoundUp(RawSize, BlockSize) / BlockSize; - ZEN_ASSERT(BlockCount <= ~uint32_t(0)); - - // Allocate the buffer for the header, metadata, and compressed blocks. - const uint64_t MetaSize = BlockCount * sizeof(uint32_t); - const uint64_t CompressedDataSize = sizeof(BufferHeader) + MetaSize + GetCompressedBlocksBound(BlockCount, BlockSize, RawSize); - UniqueBuffer CompressedData = UniqueBuffer::Alloc(CompressedDataSize); - - // Compress the raw data in blocks and store the raw data for incompressible blocks. - std::vector<uint32_t> CompressedBlockSizes; - CompressedBlockSizes.reserve(BlockCount); - uint64_t CompressedSize = 0; - { - UniqueBuffer RawBlockCopy; - MutableMemoryView CompressedBlocksView = CompressedData.GetMutableView() + sizeof(BufferHeader) + MetaSize; - - CompositeBuffer::Iterator It = RawData.GetIterator(0); - - for (uint64_t RawOffset = 0; RawOffset < RawSize;) - { - const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize); - const MemoryView RawBlock = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy); - RawHash.Append(RawBlock); - - MutableMemoryView CompressedBlock = CompressedBlocksView; - if (!CompressBlock(CompressedBlock, RawBlock)) - { - return CompositeBuffer(); - } - - uint64_t CompressedBlockSize = CompressedBlock.GetSize(); - if (RawBlockSize <= CompressedBlockSize) - { - CompressedBlockSize = RawBlockSize; - CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock); - } - else - { - CompressedBlocksView += CompressedBlockSize; - } - - CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize)); - CompressedSize += CompressedBlockSize; - RawOffset += RawBlockSize; - } - } - - // Return an uncompressed buffer if the compressed data is larger than the raw data. - if (RawSize <= MetaSize + CompressedSize) - { - CompressedData.Reset(); - return NoneEncoder().Compress(RawData, BlockSize); - } - - // Write the header and calculate the CRC-32. - for (uint32_t& Size : CompressedBlockSizes) - { - Size = ByteSwap(Size); - } - CompressedData.GetMutableView().Mid(sizeof(BufferHeader), MetaSize).CopyFrom(MakeMemoryView(CompressedBlockSizes)); - - BufferHeader Header; - Header.Method = GetMethod(); - Header.Compressor = GetCompressor(); - Header.CompressionLevel = GetCompressionLevel(); - Header.BlockSizeExponent = static_cast<uint8_t>(zen::FloorLog2_64(BlockSize)); - Header.BlockCount = static_cast<uint32_t>(BlockCount); - Header.TotalRawSize = RawSize; - Header.TotalCompressedSize = sizeof(BufferHeader) + MetaSize + CompressedSize; - Header.RawHash = RawHash.GetHash(); - Header.Write(CompressedData.GetMutableView().Left(sizeof(BufferHeader) + MetaSize)); - - const MemoryView CompositeView = CompressedData.GetView().Left(Header.TotalCompressedSize); - return CompositeBuffer(SharedBuffer::MakeView(CompositeView, CompressedData.MoveToShared())); -} - -class BlockDecoder : public BaseDecoder -{ -public: - CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const final; - [[nodiscard]] bool TryDecompressTo(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - MutableMemoryView RawView, - uint64_t RawOffset) const final; - [[nodiscard]] uint64_t GetHeaderSize(const BufferHeader& Header) const final - { - return sizeof(BufferHeader) + sizeof(uint32_t) * uint64_t(Header.BlockCount); - } - -protected: - virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0; -}; - -CompositeBuffer -BlockDecoder::Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const -{ - if (Header.BlockCount == 0 || Header.TotalCompressedSize != CompressedData.GetSize()) - { - return CompositeBuffer(); - } - - // The raw data cannot reference the compressed data unless it is owned. - // An empty raw buffer requires an empty segment, which this path creates. - if (!CompressedData.IsOwned() || Header.TotalRawSize == 0) - { - UniqueBuffer Buffer = UniqueBuffer::Alloc(Header.TotalRawSize); - return TryDecompressTo(Header, CompressedData, Buffer, 0) ? CompositeBuffer(Buffer.MoveToShared()) : CompositeBuffer(); - } - - std::vector<uint32_t> CompressedBlockSizes; - CompressedBlockSizes.resize(Header.BlockCount); - CompressedData.CopyTo(MakeMutableMemoryView(CompressedBlockSizes), sizeof(BufferHeader)); - - for (uint32_t& Size : CompressedBlockSizes) - { - Size = ByteSwap(Size); - } - - // Allocate the buffer for the raw blocks that were compressed. - SharedBuffer RawData; - MutableMemoryView RawDataView; - const uint64_t BlockSize = uint64_t(1) << Header.BlockSizeExponent; - { - uint64_t RawDataSize = 0; - uint64_t RemainingRawSize = Header.TotalRawSize; - for (const uint32_t CompressedBlockSize : CompressedBlockSizes) - { - const uint64_t RawBlockSize = zen::Min(RemainingRawSize, BlockSize); - if (CompressedBlockSize < BlockSize) - { - RawDataSize += RawBlockSize; - } - RemainingRawSize -= RawBlockSize; - } - UniqueBuffer RawDataBuffer = UniqueBuffer::Alloc(RawDataSize); - RawDataView = RawDataBuffer; - RawData = RawDataBuffer.MoveToShared(); - } - - // Decompress the compressed data in blocks and reference the uncompressed blocks. - uint64_t PendingCompressedSegmentOffset = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t); - uint64_t PendingCompressedSegmentSize = 0; - uint64_t PendingRawSegmentOffset = 0; - uint64_t PendingRawSegmentSize = 0; - std::vector<SharedBuffer> Segments; - - const auto CommitPendingCompressedSegment = - [&PendingCompressedSegmentOffset, &PendingCompressedSegmentSize, &CompressedData, &Segments] { - if (PendingCompressedSegmentSize) - { - CompressedData.IterateRange(PendingCompressedSegmentOffset, - PendingCompressedSegmentSize, - [&Segments](MemoryView View, const SharedBuffer& ViewOuter) { - Segments.push_back(SharedBuffer::MakeView(View, ViewOuter)); - }); - PendingCompressedSegmentOffset += PendingCompressedSegmentSize; - PendingCompressedSegmentSize = 0; - } - }; - - const auto CommitPendingRawSegment = [&PendingRawSegmentOffset, &PendingRawSegmentSize, &RawData, &Segments] { - if (PendingRawSegmentSize) - { - const MemoryView PendingSegment = RawData.GetView().Mid(PendingRawSegmentOffset, PendingRawSegmentSize); - Segments.push_back(SharedBuffer::MakeView(PendingSegment, RawData)); - PendingRawSegmentOffset += PendingRawSegmentSize; - PendingRawSegmentSize = 0; - } - }; - - UniqueBuffer CompressedBlockCopy; - uint64_t RemainingRawSize = Header.TotalRawSize; - uint64_t RemainingCompressedSize = CompressedData.GetSize(); - for (const uint32_t CompressedBlockSize : CompressedBlockSizes) - { - if (RemainingCompressedSize < CompressedBlockSize) - { - return CompositeBuffer(); - } - - const uint64_t RawBlockSize = zen::Min(RemainingRawSize, BlockSize); - if (RawBlockSize == CompressedBlockSize) - { - CommitPendingRawSegment(); - PendingCompressedSegmentSize += RawBlockSize; - } - else - { - CommitPendingCompressedSegment(); - const MemoryView CompressedBlock = - CompressedData.ViewOrCopyRange(PendingCompressedSegmentOffset, CompressedBlockSize, CompressedBlockCopy); - if (!DecompressBlock(RawDataView.Left(RawBlockSize), CompressedBlock)) - { - return CompositeBuffer(); - } - PendingCompressedSegmentOffset += CompressedBlockSize; - PendingRawSegmentSize += RawBlockSize; - RawDataView += RawBlockSize; - } - - RemainingCompressedSize -= CompressedBlockSize; - RemainingRawSize -= RawBlockSize; - } - - CommitPendingCompressedSegment(); - CommitPendingRawSegment(); - - return CompositeBuffer(std::move(Segments)); -} - -bool -BlockDecoder::TryDecompressTo(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - MutableMemoryView RawView, - uint64_t RawOffset) const -{ - if (Header.TotalRawSize < RawOffset + RawView.GetSize() || Header.TotalCompressedSize != CompressedData.GetSize()) - { - return false; - } - - const uint64_t BlockSize = uint64_t(1) << Header.BlockSizeExponent; - - UniqueBuffer BlockSizeBuffer; - MemoryView BlockSizeView = CompressedData.ViewOrCopyRange(sizeof(BufferHeader), Header.BlockCount * sizeof(uint32_t), BlockSizeBuffer); - std::span<uint32_t const> CompressedBlockSizes(reinterpret_cast<const uint32_t*>(BlockSizeView.GetData()), Header.BlockCount); - - UniqueBuffer CompressedBlockCopy; - UniqueBuffer UncompressedBlockCopy; - - const size_t FirstBlockIndex = uint64_t(RawOffset / BlockSize); - const size_t LastBlockIndex = uint64_t((RawOffset + RawView.GetSize() - 1) / BlockSize); - const uint64_t LastBlockSize = BlockSize - ((Header.BlockCount * BlockSize) - Header.TotalRawSize); - uint64_t OffsetInFirstBlock = RawOffset % BlockSize; - uint64_t CompressedOffset = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t); - uint64_t RemainingRawSize = RawView.GetSize(); - - for (size_t BlockIndex = 0; BlockIndex < FirstBlockIndex; BlockIndex++) - { - const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); - CompressedOffset += CompressedBlockSize; - } - - for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++) - { - const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize; - const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); - const bool IsCompressed = CompressedBlockSize < UncompressedBlockSize; - - const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock) - : zen::Min(RemainingRawSize, BlockSize); - - MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy); - - if (IsCompressed) - { - MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress); - - const bool IsAligned = BytesToUncompress == UncompressedBlockSize; - if (!IsAligned) - { - // Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries. - if (UncompressedBlockCopy.IsNull()) - { - UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize); - } - UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize); - } - - if (!DecompressBlock(UncompressedBlock, CompressedBlock)) - { - return false; - } - - if (!IsAligned) - { - RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress)); - } - } - else - { - RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress)); - } - - OffsetInFirstBlock = 0; - RemainingRawSize -= BytesToUncompress; - CompressedOffset += CompressedBlockSize; - RawView += BytesToUncompress; - } - - return RemainingRawSize == 0; -} - -////////////////////////////////////////////////////////////////////////// - -struct OodleInit -{ - OodleInit() - { - OodleConfigValues Config; - Oodle_GetConfigValues(&Config); - // Always read/write Oodle v9 binary data. - Config.m_OodleLZ_BackwardsCompatible_MajorVersion = 9; - Oodle_SetConfigValues(&Config); - } -}; - -OodleInit InitOodle; - -class OodleEncoder final : public BlockEncoder -{ -public: - OodleEncoder(OodleCompressor InCompressor, OodleCompressionLevel InCompressionLevel) - : Compressor(InCompressor) - , CompressionLevel(InCompressionLevel) - { - } - -protected: - CompressionMethod GetMethod() const final { return CompressionMethod::Oodle; } - uint8_t GetCompressor() const final { return static_cast<uint8_t>(Compressor); } - uint8_t GetCompressionLevel() const final { return static_cast<uint8_t>(CompressionLevel); } - - uint64_t CompressBlockBound(uint64_t RawSize) const final - { - return static_cast<uint64_t>(OodleLZ_GetCompressedBufferSizeNeeded(OodleLZ_Compressor_Kraken, static_cast<OO_SINTa>(RawSize))); - } - - bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const final - { - const OodleLZ_Compressor LZCompressor = GetOodleLZCompressor(Compressor); - const OodleLZ_CompressionLevel LZCompressionLevel = GetOodleLZCompressionLevel(CompressionLevel); - if (LZCompressor == OodleLZ_Compressor_Invalid || LZCompressionLevel == OodleLZ_CompressionLevel_Invalid || - LZCompressionLevel == OodleLZ_CompressionLevel_None) - { - return false; - } - - const OO_SINTa RawSize = static_cast<OO_SINTa>(RawData.GetSize()); - if (static_cast<OO_SINTa>(CompressedData.GetSize()) < OodleLZ_GetCompressedBufferSizeNeeded(LZCompressor, RawSize)) - { - return false; - } - - const OO_SINTa Size = OodleLZ_Compress(LZCompressor, RawData.GetData(), RawSize, CompressedData.GetData(), LZCompressionLevel); - CompressedData.LeftInline(static_cast<uint64_t>(Size)); - return Size > 0; - } - - static OodleLZ_Compressor GetOodleLZCompressor(OodleCompressor Compressor) - { - switch (Compressor) - { - case OodleCompressor::Selkie: - return OodleLZ_Compressor_Selkie; - case OodleCompressor::Mermaid: - return OodleLZ_Compressor_Mermaid; - case OodleCompressor::Kraken: - return OodleLZ_Compressor_Kraken; - case OodleCompressor::Leviathan: - return OodleLZ_Compressor_Leviathan; - case OodleCompressor::NotSet: - default: - return OodleLZ_Compressor_Invalid; - } - } - - static OodleLZ_CompressionLevel GetOodleLZCompressionLevel(OodleCompressionLevel Level) - { - const int IntLevel = (int)Level; - if (IntLevel < (int)OodleLZ_CompressionLevel_Min || IntLevel > (int)OodleLZ_CompressionLevel_Max) - { - return OodleLZ_CompressionLevel_Invalid; - } - return OodleLZ_CompressionLevel(IntLevel); - } - -private: - const OodleCompressor Compressor; - const OodleCompressionLevel CompressionLevel; -}; - -class OodleDecoder final : public BlockDecoder -{ -protected: - bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const final - { - const OO_SINTa RawSize = static_cast<OO_SINTa>(RawData.GetSize()); - const OO_SINTa Size = OodleLZ_Decompress(CompressedData.GetData(), - static_cast<OO_SINTa>(CompressedData.GetSize()), - RawData.GetData(), - RawSize, - OodleLZ_FuzzSafe_Yes, - OodleLZ_CheckCRC_Yes, - OodleLZ_Verbosity_None); - return Size == RawSize; - } -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class LZ4Decoder final : public BlockDecoder -{ -protected: - bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const final - { - if (CompressedData.GetSize() <= std::numeric_limits<int>::max()) - { - const int Size = LZ4_decompress_safe(static_cast<const char*>(CompressedData.GetData()), - static_cast<char*>(RawData.GetData()), - static_cast<int>(CompressedData.GetSize()), - static_cast<int>(zen::Min<uint64_t>(RawData.GetSize(), uint64_t(LZ4_MAX_INPUT_SIZE)))); - return static_cast<uint64_t>(Size) == RawData.GetSize(); - } - return false; - } -}; - -////////////////////////////////////////////////////////////////////////// - -static const BaseDecoder* -GetDecoder(CompressionMethod Method) -{ - static NoneDecoder None; - static OodleDecoder Oodle; - static LZ4Decoder LZ4; - - switch (Method) - { - default: - return nullptr; - case CompressionMethod::None: - return &None; - case CompressionMethod::Oodle: - return &Oodle; - case CompressionMethod::LZ4: - return &LZ4; - } -} - -////////////////////////////////////////////////////////////////////////// - -bool -BufferHeader::IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) -{ - uint64_t Size = CompressedData.GetSize(); - if (Size < sizeof(BufferHeader)) - { - return false; - } - const size_t StackBufferSize = 256; - uint8_t StackBuffer[StackBufferSize]; - uint64_t ReadSize = Min(Size, StackBufferSize); - BufferHeader* Header = reinterpret_cast<BufferHeader*>(StackBuffer); - { - CompositeBuffer::Iterator It; - CompressedData.CopyTo(MutableMemoryView(StackBuffer, StackBuffer + StackBufferSize), It); - } - Header->ByteSwap(); - if (Header->Magic != BufferHeader::ExpectedMagic) - { - return false; - } - const BaseDecoder* const Decoder = GetDecoder(Header->Method); - if (!Decoder) - { - return false; - } - uint32_t Crc32 = Header->Crc32; - OutRawHash = IoHash::FromBLAKE3(Header->RawHash); - OutRawSize = Header->TotalRawSize; - uint64_t HeaderSize = Decoder->GetHeaderSize(*Header); - Header->ByteSwap(); - - if (HeaderSize > ReadSize) - { - // 0.004% of cases on a Fortnite hot cache cook - UniqueBuffer HeaderCopy = UniqueBuffer::Alloc(HeaderSize); - CompositeBuffer::Iterator It; - CompressedData.CopyTo(HeaderCopy.GetMutableView(), It); - const MemoryView HeaderView = HeaderCopy.GetView(); - if (Crc32 != BufferHeader::CalculateCrc32(HeaderView)) - { - return false; - } - } - else - { - MemoryView FullHeaderView(StackBuffer, StackBuffer + HeaderSize); - if (Crc32 != BufferHeader::CalculateCrc32(FullHeaderView)) - { - return false; - } - } - return true; -} - -////////////////////////////////////////////////////////////////////////// - -template<typename BufferType> -inline CompositeBuffer -ValidBufferOrEmpty(BufferType&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) -{ - return BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize) ? CompositeBuffer(std::forward<BufferType>(CompressedData)) - : CompositeBuffer(); -} - -CompositeBuffer -CopyCompressedRange(const BufferHeader& Header, const CompositeBuffer& CompressedData, uint64_t RawOffset, uint64_t RawSize) -{ - if (Header.TotalRawSize < RawOffset + RawSize) - { - return CompositeBuffer(); - } - - if (Header.Method == CompressionMethod::None) - { - UniqueBuffer NewCompressedData = UniqueBuffer::Alloc(RawSize); - CompressedData.CopyTo(NewCompressedData.GetMutableView(), sizeof(Header) + RawOffset); - - BufferHeader NewHeader = Header; - NewHeader.Crc32 = 0; - NewHeader.TotalRawSize = RawSize; - NewHeader.TotalCompressedSize = NewHeader.TotalRawSize + sizeof(BufferHeader); - NewHeader.RawHash = BLAKE3(); - - UniqueBuffer HeaderData = UniqueBuffer::Alloc(sizeof(BufferHeader)); - NewHeader.Write(HeaderData); - - return CompositeBuffer(HeaderData.MoveToShared(), NewCompressedData.MoveToShared()); - } - else - { - UniqueBuffer BlockSizeBuffer; - MemoryView BlockSizeView = - CompressedData.ViewOrCopyRange(sizeof(BufferHeader), Header.BlockCount * sizeof(uint32_t), BlockSizeBuffer); - std::span<uint32_t const> CompressedBlockSizes(reinterpret_cast<const uint32_t*>(BlockSizeView.GetData()), Header.BlockCount); - - const uint64_t BlockSize = uint64_t(1) << Header.BlockSizeExponent; - const uint64_t LastBlockSize = BlockSize - ((Header.BlockCount * BlockSize) - Header.TotalRawSize); - const size_t FirstBlock = uint64_t(RawOffset / BlockSize); - const size_t LastBlock = uint64_t((RawOffset + RawSize - 1) / BlockSize); - uint64_t CompressedOffset = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t); - - const uint64_t NewBlockCount = LastBlock - FirstBlock + 1; - const uint64_t NewMetaSize = NewBlockCount * sizeof(uint32_t); - uint64_t NewCompressedSize = 0; - uint64_t NewTotalRawSize = 0; - std::vector<uint32_t> NewCompressedBlockSizes; - - NewCompressedBlockSizes.reserve(NewBlockCount); - for (size_t BlockIndex = FirstBlock; BlockIndex <= LastBlock; ++BlockIndex) - { - const uint64_t UncompressedBlockSize = (BlockIndex == Header.BlockCount - 1) ? LastBlockSize : BlockSize; - NewTotalRawSize += UncompressedBlockSize; - - const uint32_t CompressedBlockSize = CompressedBlockSizes[BlockIndex]; - NewCompressedBlockSizes.push_back(CompressedBlockSize); - NewCompressedSize += ByteSwap(CompressedBlockSize); - } - - const uint64_t NewTotalCompressedSize = sizeof(BufferHeader) + NewBlockCount * sizeof(uint32_t) + NewCompressedSize; - UniqueBuffer NewCompressedData = UniqueBuffer::Alloc(NewTotalCompressedSize); - MutableMemoryView NewCompressedBlocks = NewCompressedData.GetMutableView() + sizeof(BufferHeader) + NewMetaSize; - - // Seek to first compressed block - for (size_t BlockIndex = 0; BlockIndex < FirstBlock; ++BlockIndex) - { - const uint64_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); - CompressedOffset += CompressedBlockSize; - } - - // Copy blocks - UniqueBuffer CompressedBlockCopy; - const MemoryView CompressedRange = CompressedData.ViewOrCopyRange(CompressedOffset, NewCompressedSize, CompressedBlockCopy); - NewCompressedBlocks.CopyFrom(CompressedRange); - - // Copy block sizes - NewCompressedData.GetMutableView().Mid(sizeof(BufferHeader), NewMetaSize).CopyFrom(MakeMemoryView(NewCompressedBlockSizes)); - - BufferHeader NewHeader; - NewHeader.Crc32 = 0; - NewHeader.Method = Header.Method; - NewHeader.Compressor = Header.Compressor; - NewHeader.CompressionLevel = Header.CompressionLevel; - NewHeader.BlockSizeExponent = Header.BlockSizeExponent; - NewHeader.BlockCount = static_cast<uint32_t>(NewBlockCount); - NewHeader.TotalRawSize = NewTotalRawSize; - NewHeader.TotalCompressedSize = NewTotalCompressedSize; - NewHeader.RawHash = BLAKE3(); - NewHeader.Write(NewCompressedData.GetMutableView().Left(sizeof(BufferHeader) + NewMetaSize)); - - return CompositeBuffer(NewCompressedData.MoveToShared()); - } -} - -} // namespace zen::detail - -namespace zen { - -const CompressedBuffer CompressedBuffer::Null; - -CompressedBuffer -CompressedBuffer::Compress(const CompositeBuffer& RawData, - OodleCompressor Compressor, - OodleCompressionLevel CompressionLevel, - uint64_t BlockSize) -{ - using namespace detail; - - if (BlockSize == 0) - { - BlockSize = DefaultBlockSize; - } - - CompressedBuffer Local; - if (CompressionLevel == OodleCompressionLevel::None) - { - Local.CompressedData = NoneEncoder().Compress(RawData, BlockSize); - } - else - { - Local.CompressedData = OodleEncoder(Compressor, CompressionLevel).Compress(RawData, BlockSize); - } - return Local; -} - -CompressedBuffer -CompressedBuffer::Compress(const SharedBuffer& RawData, - OodleCompressor Compressor, - OodleCompressionLevel CompressionLevel, - uint64_t BlockSize) -{ - return Compress(CompositeBuffer(RawData), Compressor, CompressionLevel, BlockSize); -} - -CompressedBuffer -CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) -{ - CompressedBuffer Local; - Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize); - return Local; -} - -CompressedBuffer -CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) -{ - CompressedBuffer Local; - Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize); - return Local; -} - -CompressedBuffer -CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) -{ - CompressedBuffer Local; - Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize); - return Local; -} - -CompressedBuffer -CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) -{ - CompressedBuffer Local; - Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize); - return Local; -} - -CompressedBuffer -CompressedBuffer::FromCompressedNoValidate(IoBuffer&& InCompressedData) -{ - if (InCompressedData.GetSize() <= sizeof(detail::BufferHeader)) - { - return CompressedBuffer(); - } - CompressedBuffer Local; - Local.CompressedData = CompositeBuffer(SharedBuffer(std::move(InCompressedData))); - return Local; -} - -CompressedBuffer -CompressedBuffer::FromCompressedNoValidate(CompositeBuffer&& InCompressedData) -{ - if (InCompressedData.GetSize() <= sizeof(detail::BufferHeader)) - { - return CompressedBuffer(); - } - CompressedBuffer Local; - Local.CompressedData = std::move(InCompressedData); - return Local; -} - -bool -CompressedBuffer::ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) -{ - return detail::BufferHeader::IsValid(SharedBuffer(std::move(CompressedData)), OutRawHash, OutRawSize); -} - -bool -CompressedBuffer::ValidateCompressedHeader(const IoBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) -{ - return detail::BufferHeader::IsValid(SharedBuffer(CompressedData), OutRawHash, OutRawSize); -} - -uint64_t -CompressedBuffer::DecodeRawSize() const -{ - return CompressedData ? detail::BufferHeader::Read(CompressedData).TotalRawSize : 0; -} - -IoHash -CompressedBuffer::DecodeRawHash() const -{ - return CompressedData ? IoHash::FromBLAKE3(detail::BufferHeader::Read(CompressedData).RawHash) : IoHash(); -} - -CompressedBuffer -CompressedBuffer::CopyRange(uint64_t RawOffset, uint64_t RawSize) const -{ - using namespace detail; - const BufferHeader Header = BufferHeader::Read(CompressedData); - const uint64_t TotalRawSize = RawSize < ~uint64_t(0) ? RawSize : Header.TotalRawSize - RawOffset; - - CompressedBuffer Range; - Range.CompressedData = CopyCompressedRange(Header, CompressedData, RawOffset, TotalRawSize); - - return Range; -} - -bool -CompressedBuffer::TryDecompressTo(MutableMemoryView RawView, uint64_t RawOffset) const -{ - using namespace detail; - if (CompressedData) - { - const BufferHeader Header = BufferHeader::Read(CompressedData); - if (Header.Magic == BufferHeader::ExpectedMagic) - { - if (const BaseDecoder* const Decoder = GetDecoder(Header.Method)) - { - return Decoder->TryDecompressTo(Header, CompressedData, RawView, RawOffset); - } - } - } - return false; -} - -SharedBuffer -CompressedBuffer::Decompress(uint64_t RawOffset, uint64_t RawSize) const -{ - using namespace detail; - if (CompressedData && RawSize > 0) - { - const BufferHeader Header = BufferHeader::Read(CompressedData); - if (Header.Magic == BufferHeader::ExpectedMagic) - { - if (const BaseDecoder* const Decoder = GetDecoder(Header.Method)) - { - const uint64_t TotalRawSize = RawSize < ~uint64_t(0) ? RawSize : Header.TotalRawSize - RawOffset; - UniqueBuffer RawData = UniqueBuffer::Alloc(TotalRawSize); - if (Decoder->TryDecompressTo(Header, CompressedData, RawData, RawOffset)) - { - return RawData.MoveToShared(); - } - } - } - } - return SharedBuffer(); -} - -CompositeBuffer -CompressedBuffer::DecompressToComposite() const -{ - using namespace detail; - if (CompressedData) - { - const BufferHeader Header = BufferHeader::Read(CompressedData); - if (Header.Magic == BufferHeader::ExpectedMagic) - { - if (const BaseDecoder* const Decoder = GetDecoder(Header.Method)) - { - return Decoder->Decompress(Header, CompressedData); - } - } - } - return CompositeBuffer(); -} - -bool -CompressedBuffer::TryGetCompressParameters(OodleCompressor& OutCompressor, - OodleCompressionLevel& OutCompressionLevel, - uint64_t& OutBlockSize) const -{ - using namespace detail; - if (CompressedData) - { - switch (const BufferHeader Header = BufferHeader::Read(CompressedData); Header.Method) - { - case CompressionMethod::None: - OutCompressor = OodleCompressor::NotSet; - OutCompressionLevel = OodleCompressionLevel::None; - OutBlockSize = 0; - return true; - case CompressionMethod::Oodle: - OutCompressor = OodleCompressor(Header.Compressor); - OutCompressionLevel = OodleCompressionLevel(Header.CompressionLevel); - OutBlockSize = uint64_t(1) << Header.BlockSizeExponent; - return true; - default: - break; - } - } - return false; -} - -/** - ______________________ _____________________________ - \__ ___/\_ _____// _____/\__ ___/ _____/ - | | | __)_ \_____ \ | | \_____ \ - | | | \/ \ | | / \ - |____| /_______ /_______ / |____| /_______ / - \/ \/ \/ - */ - -#if ZEN_WITH_TESTS - -TEST_CASE("CompressedBuffer") -{ - uint8_t Zeroes[1024]{}; - uint8_t Ones[1024]; - memset(Ones, 1, sizeof Ones); - - { - CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))), - OodleCompressor::NotSet, - OodleCompressionLevel::None); - - CHECK(Buffer.DecodeRawSize() == sizeof(Zeroes)); - CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(detail::BufferHeader))); - - CompositeBuffer Compressed = Buffer.GetCompressed(); - IoHash DecodedHash; - uint64_t DecodedRawSize; - CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize); - - CHECK(BufferD.IsNull() == false); - - CompositeBuffer Decomp = BufferD.DecompressToComposite(); - - CHECK(Decomp.GetSize() == DecodedRawSize); - CHECK(IoHash::HashBuffer(Decomp) == DecodedHash); - } - - { - CompressedBuffer Buffer = CompressedBuffer::Compress( - CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones))), - OodleCompressor::NotSet, - OodleCompressionLevel::None); - - CHECK(Buffer.DecodeRawSize() == (sizeof(Zeroes) + sizeof(Ones))); - CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(Ones) + sizeof(detail::BufferHeader))); - - CompositeBuffer Compressed = Buffer.GetCompressed(); - IoHash DecodedHash; - uint64_t DecodedRawSize; - CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize); - - CHECK(BufferD.IsNull() == false); - - CompositeBuffer Decomp = BufferD.DecompressToComposite(); - - CHECK(Decomp.GetSize() == DecodedRawSize); - CHECK(IoHash::HashBuffer(Decomp) == DecodedHash); - } - - { - CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)))); - - CHECK(Buffer.DecodeRawSize() == sizeof(Zeroes)); - CHECK(Buffer.GetCompressedSize() < sizeof(Zeroes)); - - CompositeBuffer Compressed = Buffer.GetCompressed(); - IoHash DecodedHash; - uint64_t DecodedRawSize; - CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize); - - CHECK(BufferD.IsNull() == false); - - CompositeBuffer Decomp = BufferD.DecompressToComposite(); - - CHECK(Decomp.GetSize() == DecodedRawSize); - CHECK(IoHash::HashBuffer(Decomp) == DecodedHash); - } - - { - CompressedBuffer Buffer = CompressedBuffer::Compress( - CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones)))); - - CHECK(Buffer.DecodeRawSize() == (sizeof(Zeroes) + sizeof(Ones))); - CHECK(Buffer.GetCompressedSize() < (sizeof(Zeroes) + sizeof(Ones))); - - CompositeBuffer Compressed = Buffer.GetCompressed(); - IoHash DecodedHash; - uint64_t DecodedRawSize; - CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize); - - CHECK(BufferD.IsNull() == false); - - CompositeBuffer Decomp = BufferD.DecompressToComposite(); - - CHECK(Decomp.GetSize() == DecodedRawSize); - CHECK(IoHash::HashBuffer(Decomp) == DecodedHash); - } - - auto GenerateData = [](uint64_t N) -> std::vector<uint64_t> { - std::vector<uint64_t> Data; - Data.resize(N); - for (size_t Idx = 0; Idx < Data.size(); ++Idx) - { - Data[Idx] = Idx; - } - return Data; - }; - - auto ValidateData = [](std::span<uint64_t const> Values, std::span<uint64_t const> ExpectedValues, uint64_t Offset) { - for (size_t Idx = Offset; uint64_t Value : Values) - { - const uint64_t ExpectedValue = ExpectedValues[Idx++]; - CHECK(Value == ExpectedValue); - } - }; - - SUBCASE("decompress with offset and size") - { - auto UncompressAndValidate = [&ValidateData](CompressedBuffer Compressed, - uint64_t OffsetCount, - uint64_t Count, - const std::vector<uint64_t>& ExpectedValues) { - SharedBuffer Uncompressed = Compressed.Decompress(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)); - CHECK(Uncompressed.GetSize() == Count * sizeof(uint64_t)); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - ValidateData(Values, ExpectedValues, OffsetCount); - }; - - const uint64_t BlockSize = 64 * sizeof(uint64_t); - const uint64_t N = 5000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), - OodleCompressor::Mermaid, - OodleCompressionLevel::Optimal4, - BlockSize); - UncompressAndValidate(Compressed, 0, N, ExpectedValues); - UncompressAndValidate(Compressed, 1, N - 1, ExpectedValues); - UncompressAndValidate(Compressed, N - 1, 1, ExpectedValues); - UncompressAndValidate(Compressed, 0, 1, ExpectedValues); - UncompressAndValidate(Compressed, 2, 4, ExpectedValues); - UncompressAndValidate(Compressed, 0, 512, ExpectedValues); - UncompressAndValidate(Compressed, 3, 514, ExpectedValues); - UncompressAndValidate(Compressed, 256, 512, ExpectedValues); - UncompressAndValidate(Compressed, 512, 512, ExpectedValues); - } - - SUBCASE("decompress with offset only") - { - const uint64_t BlockSize = 64 * sizeof(uint64_t); - const uint64_t N = 1000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), - OodleCompressor::Mermaid, - OodleCompressionLevel::Optimal4, - BlockSize); - const uint64_t OffsetCount = 150; - SharedBuffer Uncompressed = Compressed.Decompress(OffsetCount * sizeof(uint64_t)); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - ValidateData(Values, ExpectedValues, OffsetCount); - } - - SUBCASE("decompress buffer with one block") - { - const uint64_t BlockSize = 256 * sizeof(uint64_t); - const uint64_t N = 100; - std::vector<uint64_t> ExpectedValues = GenerateData(N); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), - OodleCompressor::Mermaid, - OodleCompressionLevel::Optimal4, - BlockSize); - const uint64_t OffsetCount = 2; - const uint64_t Count = 50; - SharedBuffer Uncompressed = Compressed.Decompress(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - ValidateData(Values, ExpectedValues, OffsetCount); - } - - SUBCASE("decompress uncompressed buffer") - { - const uint64_t N = 4242; - std::vector<uint64_t> ExpectedValues = GenerateData(N); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), - OodleCompressor::NotSet, - OodleCompressionLevel::None); - { - const uint64_t OffsetCount = 0; - const uint64_t Count = N; - SharedBuffer Uncompressed = Compressed.Decompress(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - ValidateData(Values, ExpectedValues, OffsetCount); - } - - { - const uint64_t OffsetCount = 21; - const uint64_t Count = 999; - SharedBuffer Uncompressed = Compressed.Decompress(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - ValidateData(Values, ExpectedValues, OffsetCount); - } - } - - SUBCASE("copy range") - { - const uint64_t BlockSize = 64 * sizeof(uint64_t); - const uint64_t N = 1000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); - - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), - OodleCompressor::Mermaid, - OodleCompressionLevel::Optimal4, - BlockSize); - - { - const uint64_t OffsetCount = 0; - const uint64_t Count = N; - SharedBuffer Uncompressed = Compressed.CopyRange(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)).Decompress(); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - CHECK(Values.size() == Count); - ValidateData(Values, ExpectedValues, OffsetCount); - } - - { - const uint64_t OffsetCount = 64; - const uint64_t Count = N - 64; - SharedBuffer Uncompressed = Compressed.CopyRange(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)).Decompress(); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - CHECK(Values.size() == Count); - ValidateData(Values, ExpectedValues, OffsetCount); - } - - { - const uint64_t OffsetCount = 64 * 2 + 32; - const uint64_t Count = N - OffsetCount; - const uint64_t RawOffset = OffsetCount * sizeof(uint64_t); - const uint64_t RawSize = Count * sizeof(uint64_t); - uint64_t FirstBlockOffset = RawOffset % BlockSize; - - SharedBuffer Uncompressed = Compressed.CopyRange(RawOffset, RawSize).Decompress(); - std::span<uint64_t const> AllValues((const uint64_t*)Uncompressed.GetData(), RawSize / sizeof(uint64_t)); - std::span<uint64_t const> Values((const uint64_t*)(((const uint8_t*)(Uncompressed.GetData()) + FirstBlockOffset)), - RawSize / sizeof(uint64_t)); - CHECK(Values.size() == Count); - ValidateData(Values, ExpectedValues, OffsetCount); - } - - { - const uint64_t OffsetCount = 64 * 2 + 63; - const uint64_t Count = N - OffsetCount - 5; - const uint64_t RawOffset = OffsetCount * sizeof(uint64_t); - const uint64_t RawSize = Count * sizeof(uint64_t); - uint64_t FirstBlockOffset = RawOffset % BlockSize; - - SharedBuffer Uncompressed = Compressed.CopyRange(RawOffset, RawSize).Decompress(); - std::span<uint64_t const> AllValues((const uint64_t*)Uncompressed.GetData(), RawSize / sizeof(uint64_t)); - std::span<uint64_t const> Values((const uint64_t*)(((const uint8_t*)(Uncompressed.GetData()) + FirstBlockOffset)), - RawSize / sizeof(uint64_t)); - CHECK(Values.size() == Count); - ValidateData(Values, ExpectedValues, OffsetCount); - } - } - - SUBCASE("copy uncompressed range") - { - const uint64_t N = 1000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); - - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), - OodleCompressor::NotSet, - OodleCompressionLevel::None); - - { - const uint64_t OffsetCount = 0; - const uint64_t Count = N; - SharedBuffer Uncompressed = Compressed.CopyRange(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)).Decompress(); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - CHECK(Values.size() == Count); - ValidateData(Values, ExpectedValues, OffsetCount); - } - - { - const uint64_t OffsetCount = 1; - const uint64_t Count = N - OffsetCount; - SharedBuffer Uncompressed = Compressed.CopyRange(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)).Decompress(); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - CHECK(Values.size() == Count); - ValidateData(Values, ExpectedValues, OffsetCount); - } - - { - const uint64_t OffsetCount = 42; - const uint64_t Count = 100; - SharedBuffer Uncompressed = Compressed.CopyRange(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)).Decompress(); - std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); - CHECK(Values.size() == Count); - ValidateData(Values, ExpectedValues, OffsetCount); - } - } -} - -void -compress_forcelink() -{ -} -#endif - -} // namespace zen diff --git a/zencore/crc32.cpp b/zencore/crc32.cpp deleted file mode 100644 index d4a3cac57..000000000 --- a/zencore/crc32.cpp +++ /dev/null @@ -1,545 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/crc32.h" - -namespace CRC32 { - -static const uint32_t CRCTable_DEPRECATED[256] = { - 0x00000000, 0x04C11DB7, 0x09823B6E, 0x0D4326D9, 0x130476DC, 0x17C56B6B, 0x1A864DB2, 0x1E475005, 0x2608EDB8, 0x22C9F00F, 0x2F8AD6D6, - 0x2B4BCB61, 0x350C9B64, 0x31CD86D3, 0x3C8EA00A, 0x384FBDBD, 0x4C11DB70, 0x48D0C6C7, 0x4593E01E, 0x4152FDA9, 0x5F15ADAC, 0x5BD4B01B, - 0x569796C2, 0x52568B75, 0x6A1936C8, 0x6ED82B7F, 0x639B0DA6, 0x675A1011, 0x791D4014, 0x7DDC5DA3, 0x709F7B7A, 0x745E66CD, 0x9823B6E0, - 0x9CE2AB57, 0x91A18D8E, 0x95609039, 0x8B27C03C, 0x8FE6DD8B, 0x82A5FB52, 0x8664E6E5, 0xBE2B5B58, 0xBAEA46EF, 0xB7A96036, 0xB3687D81, - 0xAD2F2D84, 0xA9EE3033, 0xA4AD16EA, 0xA06C0B5D, 0xD4326D90, 0xD0F37027, 0xDDB056FE, 0xD9714B49, 0xC7361B4C, 0xC3F706FB, 0xCEB42022, - 0xCA753D95, 0xF23A8028, 0xF6FB9D9F, 0xFBB8BB46, 0xFF79A6F1, 0xE13EF6F4, 0xE5FFEB43, 0xE8BCCD9A, 0xEC7DD02D, 0x34867077, 0x30476DC0, - 0x3D044B19, 0x39C556AE, 0x278206AB, 0x23431B1C, 0x2E003DC5, 0x2AC12072, 0x128E9DCF, 0x164F8078, 0x1B0CA6A1, 0x1FCDBB16, 0x018AEB13, - 0x054BF6A4, 0x0808D07D, 0x0CC9CDCA, 0x7897AB07, 0x7C56B6B0, 0x71159069, 0x75D48DDE, 0x6B93DDDB, 0x6F52C06C, 0x6211E6B5, 0x66D0FB02, - 0x5E9F46BF, 0x5A5E5B08, 0x571D7DD1, 0x53DC6066, 0x4D9B3063, 0x495A2DD4, 0x44190B0D, 0x40D816BA, 0xACA5C697, 0xA864DB20, 0xA527FDF9, - 0xA1E6E04E, 0xBFA1B04B, 0xBB60ADFC, 0xB6238B25, 0xB2E29692, 0x8AAD2B2F, 0x8E6C3698, 0x832F1041, 0x87EE0DF6, 0x99A95DF3, 0x9D684044, - 0x902B669D, 0x94EA7B2A, 0xE0B41DE7, 0xE4750050, 0xE9362689, 0xEDF73B3E, 0xF3B06B3B, 0xF771768C, 0xFA325055, 0xFEF34DE2, 0xC6BCF05F, - 0xC27DEDE8, 0xCF3ECB31, 0xCBFFD686, 0xD5B88683, 0xD1799B34, 0xDC3ABDED, 0xD8FBA05A, 0x690CE0EE, 0x6DCDFD59, 0x608EDB80, 0x644FC637, - 0x7A089632, 0x7EC98B85, 0x738AAD5C, 0x774BB0EB, 0x4F040D56, 0x4BC510E1, 0x46863638, 0x42472B8F, 0x5C007B8A, 0x58C1663D, 0x558240E4, - 0x51435D53, 0x251D3B9E, 0x21DC2629, 0x2C9F00F0, 0x285E1D47, 0x36194D42, 0x32D850F5, 0x3F9B762C, 0x3B5A6B9B, 0x0315D626, 0x07D4CB91, - 0x0A97ED48, 0x0E56F0FF, 0x1011A0FA, 0x14D0BD4D, 0x19939B94, 0x1D528623, 0xF12F560E, 0xF5EE4BB9, 0xF8AD6D60, 0xFC6C70D7, 0xE22B20D2, - 0xE6EA3D65, 0xEBA91BBC, 0xEF68060B, 0xD727BBB6, 0xD3E6A601, 0xDEA580D8, 0xDA649D6F, 0xC423CD6A, 0xC0E2D0DD, 0xCDA1F604, 0xC960EBB3, - 0xBD3E8D7E, 0xB9FF90C9, 0xB4BCB610, 0xB07DABA7, 0xAE3AFBA2, 0xAAFBE615, 0xA7B8C0CC, 0xA379DD7B, 0x9B3660C6, 0x9FF77D71, 0x92B45BA8, - 0x9675461F, 0x8832161A, 0x8CF30BAD, 0x81B02D74, 0x857130C3, 0x5D8A9099, 0x594B8D2E, 0x5408ABF7, 0x50C9B640, 0x4E8EE645, 0x4A4FFBF2, - 0x470CDD2B, 0x43CDC09C, 0x7B827D21, 0x7F436096, 0x7200464F, 0x76C15BF8, 0x68860BFD, 0x6C47164A, 0x61043093, 0x65C52D24, 0x119B4BE9, - 0x155A565E, 0x18197087, 0x1CD86D30, 0x029F3D35, 0x065E2082, 0x0B1D065B, 0x0FDC1BEC, 0x3793A651, 0x3352BBE6, 0x3E119D3F, 0x3AD08088, - 0x2497D08D, 0x2056CD3A, 0x2D15EBE3, 0x29D4F654, 0xC5A92679, 0xC1683BCE, 0xCC2B1D17, 0xC8EA00A0, 0xD6AD50A5, 0xD26C4D12, 0xDF2F6BCB, - 0xDBEE767C, 0xE3A1CBC1, 0xE760D676, 0xEA23F0AF, 0xEEE2ED18, 0xF0A5BD1D, 0xF464A0AA, 0xF9278673, 0xFDE69BC4, 0x89B8FD09, 0x8D79E0BE, - 0x803AC667, 0x84FBDBD0, 0x9ABC8BD5, 0x9E7D9662, 0x933EB0BB, 0x97FFAD0C, 0xAFB010B1, 0xAB710D06, 0xA6322BDF, 0xA2F33668, 0xBCB4666D, - 0xB8757BDA, 0xB5365D03, 0xB1F740B4}; - -static const uint32_t CRCTablesSB8_DEPRECATED[8][256] = { - {0x00000000, 0xb71dc104, 0x6e3b8209, 0xd926430d, 0xdc760413, 0x6b6bc517, 0xb24d861a, 0x0550471e, 0xb8ed0826, 0x0ff0c922, 0xd6d68a2f, - 0x61cb4b2b, 0x649b0c35, 0xd386cd31, 0x0aa08e3c, 0xbdbd4f38, 0x70db114c, 0xc7c6d048, 0x1ee09345, 0xa9fd5241, 0xacad155f, 0x1bb0d45b, - 0xc2969756, 0x758b5652, 0xc836196a, 0x7f2bd86e, 0xa60d9b63, 0x11105a67, 0x14401d79, 0xa35ddc7d, 0x7a7b9f70, 0xcd665e74, 0xe0b62398, - 0x57abe29c, 0x8e8da191, 0x39906095, 0x3cc0278b, 0x8bdde68f, 0x52fba582, 0xe5e66486, 0x585b2bbe, 0xef46eaba, 0x3660a9b7, 0x817d68b3, - 0x842d2fad, 0x3330eea9, 0xea16ada4, 0x5d0b6ca0, 0x906d32d4, 0x2770f3d0, 0xfe56b0dd, 0x494b71d9, 0x4c1b36c7, 0xfb06f7c3, 0x2220b4ce, - 0x953d75ca, 0x28803af2, 0x9f9dfbf6, 0x46bbb8fb, 0xf1a679ff, 0xf4f63ee1, 0x43ebffe5, 0x9acdbce8, 0x2dd07dec, 0x77708634, 0xc06d4730, - 0x194b043d, 0xae56c539, 0xab068227, 0x1c1b4323, 0xc53d002e, 0x7220c12a, 0xcf9d8e12, 0x78804f16, 0xa1a60c1b, 0x16bbcd1f, 0x13eb8a01, - 0xa4f64b05, 0x7dd00808, 0xcacdc90c, 0x07ab9778, 0xb0b6567c, 0x69901571, 0xde8dd475, 0xdbdd936b, 0x6cc0526f, 0xb5e61162, 0x02fbd066, - 0xbf469f5e, 0x085b5e5a, 0xd17d1d57, 0x6660dc53, 0x63309b4d, 0xd42d5a49, 0x0d0b1944, 0xba16d840, 0x97c6a5ac, 0x20db64a8, 0xf9fd27a5, - 0x4ee0e6a1, 0x4bb0a1bf, 0xfcad60bb, 0x258b23b6, 0x9296e2b2, 0x2f2bad8a, 0x98366c8e, 0x41102f83, 0xf60dee87, 0xf35da999, 0x4440689d, - 0x9d662b90, 0x2a7bea94, 0xe71db4e0, 0x500075e4, 0x892636e9, 0x3e3bf7ed, 0x3b6bb0f3, 0x8c7671f7, 0x555032fa, 0xe24df3fe, 0x5ff0bcc6, - 0xe8ed7dc2, 0x31cb3ecf, 0x86d6ffcb, 0x8386b8d5, 0x349b79d1, 0xedbd3adc, 0x5aa0fbd8, 0xeee00c69, 0x59fdcd6d, 0x80db8e60, 0x37c64f64, - 0x3296087a, 0x858bc97e, 0x5cad8a73, 0xebb04b77, 0x560d044f, 0xe110c54b, 0x38368646, 0x8f2b4742, 0x8a7b005c, 0x3d66c158, 0xe4408255, - 0x535d4351, 0x9e3b1d25, 0x2926dc21, 0xf0009f2c, 0x471d5e28, 0x424d1936, 0xf550d832, 0x2c769b3f, 0x9b6b5a3b, 0x26d61503, 0x91cbd407, - 0x48ed970a, 0xfff0560e, 0xfaa01110, 0x4dbdd014, 0x949b9319, 0x2386521d, 0x0e562ff1, 0xb94beef5, 0x606dadf8, 0xd7706cfc, 0xd2202be2, - 0x653deae6, 0xbc1ba9eb, 0x0b0668ef, 0xb6bb27d7, 0x01a6e6d3, 0xd880a5de, 0x6f9d64da, 0x6acd23c4, 0xddd0e2c0, 0x04f6a1cd, 0xb3eb60c9, - 0x7e8d3ebd, 0xc990ffb9, 0x10b6bcb4, 0xa7ab7db0, 0xa2fb3aae, 0x15e6fbaa, 0xccc0b8a7, 0x7bdd79a3, 0xc660369b, 0x717df79f, 0xa85bb492, - 0x1f467596, 0x1a163288, 0xad0bf38c, 0x742db081, 0xc3307185, 0x99908a5d, 0x2e8d4b59, 0xf7ab0854, 0x40b6c950, 0x45e68e4e, 0xf2fb4f4a, - 0x2bdd0c47, 0x9cc0cd43, 0x217d827b, 0x9660437f, 0x4f460072, 0xf85bc176, 0xfd0b8668, 0x4a16476c, 0x93300461, 0x242dc565, 0xe94b9b11, - 0x5e565a15, 0x87701918, 0x306dd81c, 0x353d9f02, 0x82205e06, 0x5b061d0b, 0xec1bdc0f, 0x51a69337, 0xe6bb5233, 0x3f9d113e, 0x8880d03a, - 0x8dd09724, 0x3acd5620, 0xe3eb152d, 0x54f6d429, 0x7926a9c5, 0xce3b68c1, 0x171d2bcc, 0xa000eac8, 0xa550add6, 0x124d6cd2, 0xcb6b2fdf, - 0x7c76eedb, 0xc1cba1e3, 0x76d660e7, 0xaff023ea, 0x18ede2ee, 0x1dbda5f0, 0xaaa064f4, 0x738627f9, 0xc49be6fd, 0x09fdb889, 0xbee0798d, - 0x67c63a80, 0xd0dbfb84, 0xd58bbc9a, 0x62967d9e, 0xbbb03e93, 0x0cadff97, 0xb110b0af, 0x060d71ab, 0xdf2b32a6, 0x6836f3a2, 0x6d66b4bc, - 0xda7b75b8, 0x035d36b5, 0xb440f7b1}, - {0x00000000, 0xdcc119d2, 0x0f9ef2a0, 0xd35feb72, 0xa9212445, 0x75e03d97, 0xa6bfd6e5, 0x7a7ecf37, 0x5243488a, 0x8e825158, 0x5dddba2a, - 0x811ca3f8, 0xfb626ccf, 0x27a3751d, 0xf4fc9e6f, 0x283d87bd, 0x139b5110, 0xcf5a48c2, 0x1c05a3b0, 0xc0c4ba62, 0xbaba7555, 0x667b6c87, - 0xb52487f5, 0x69e59e27, 0x41d8199a, 0x9d190048, 0x4e46eb3a, 0x9287f2e8, 0xe8f93ddf, 0x3438240d, 0xe767cf7f, 0x3ba6d6ad, 0x2636a320, - 0xfaf7baf2, 0x29a85180, 0xf5694852, 0x8f178765, 0x53d69eb7, 0x808975c5, 0x5c486c17, 0x7475ebaa, 0xa8b4f278, 0x7beb190a, 0xa72a00d8, - 0xdd54cfef, 0x0195d63d, 0xd2ca3d4f, 0x0e0b249d, 0x35adf230, 0xe96cebe2, 0x3a330090, 0xe6f21942, 0x9c8cd675, 0x404dcfa7, 0x931224d5, - 0x4fd33d07, 0x67eebaba, 0xbb2fa368, 0x6870481a, 0xb4b151c8, 0xcecf9eff, 0x120e872d, 0xc1516c5f, 0x1d90758d, 0x4c6c4641, 0x90ad5f93, - 0x43f2b4e1, 0x9f33ad33, 0xe54d6204, 0x398c7bd6, 0xead390a4, 0x36128976, 0x1e2f0ecb, 0xc2ee1719, 0x11b1fc6b, 0xcd70e5b9, 0xb70e2a8e, - 0x6bcf335c, 0xb890d82e, 0x6451c1fc, 0x5ff71751, 0x83360e83, 0x5069e5f1, 0x8ca8fc23, 0xf6d63314, 0x2a172ac6, 0xf948c1b4, 0x2589d866, - 0x0db45fdb, 0xd1754609, 0x022aad7b, 0xdeebb4a9, 0xa4957b9e, 0x7854624c, 0xab0b893e, 0x77ca90ec, 0x6a5ae561, 0xb69bfcb3, 0x65c417c1, - 0xb9050e13, 0xc37bc124, 0x1fbad8f6, 0xcce53384, 0x10242a56, 0x3819adeb, 0xe4d8b439, 0x37875f4b, 0xeb464699, 0x913889ae, 0x4df9907c, - 0x9ea67b0e, 0x426762dc, 0x79c1b471, 0xa500ada3, 0x765f46d1, 0xaa9e5f03, 0xd0e09034, 0x0c2189e6, 0xdf7e6294, 0x03bf7b46, 0x2b82fcfb, - 0xf743e529, 0x241c0e5b, 0xf8dd1789, 0x82a3d8be, 0x5e62c16c, 0x8d3d2a1e, 0x51fc33cc, 0x98d88c82, 0x44199550, 0x97467e22, 0x4b8767f0, - 0x31f9a8c7, 0xed38b115, 0x3e675a67, 0xe2a643b5, 0xca9bc408, 0x165addda, 0xc50536a8, 0x19c42f7a, 0x63bae04d, 0xbf7bf99f, 0x6c2412ed, - 0xb0e50b3f, 0x8b43dd92, 0x5782c440, 0x84dd2f32, 0x581c36e0, 0x2262f9d7, 0xfea3e005, 0x2dfc0b77, 0xf13d12a5, 0xd9009518, 0x05c18cca, - 0xd69e67b8, 0x0a5f7e6a, 0x7021b15d, 0xace0a88f, 0x7fbf43fd, 0xa37e5a2f, 0xbeee2fa2, 0x622f3670, 0xb170dd02, 0x6db1c4d0, 0x17cf0be7, - 0xcb0e1235, 0x1851f947, 0xc490e095, 0xecad6728, 0x306c7efa, 0xe3339588, 0x3ff28c5a, 0x458c436d, 0x994d5abf, 0x4a12b1cd, 0x96d3a81f, - 0xad757eb2, 0x71b46760, 0xa2eb8c12, 0x7e2a95c0, 0x04545af7, 0xd8954325, 0x0bcaa857, 0xd70bb185, 0xff363638, 0x23f72fea, 0xf0a8c498, - 0x2c69dd4a, 0x5617127d, 0x8ad60baf, 0x5989e0dd, 0x8548f90f, 0xd4b4cac3, 0x0875d311, 0xdb2a3863, 0x07eb21b1, 0x7d95ee86, 0xa154f754, - 0x720b1c26, 0xaeca05f4, 0x86f78249, 0x5a369b9b, 0x896970e9, 0x55a8693b, 0x2fd6a60c, 0xf317bfde, 0x204854ac, 0xfc894d7e, 0xc72f9bd3, - 0x1bee8201, 0xc8b16973, 0x147070a1, 0x6e0ebf96, 0xb2cfa644, 0x61904d36, 0xbd5154e4, 0x956cd359, 0x49adca8b, 0x9af221f9, 0x4633382b, - 0x3c4df71c, 0xe08ceece, 0x33d305bc, 0xef121c6e, 0xf28269e3, 0x2e437031, 0xfd1c9b43, 0x21dd8291, 0x5ba34da6, 0x87625474, 0x543dbf06, - 0x88fca6d4, 0xa0c12169, 0x7c0038bb, 0xaf5fd3c9, 0x739eca1b, 0x09e0052c, 0xd5211cfe, 0x067ef78c, 0xdabfee5e, 0xe11938f3, 0x3dd82121, - 0xee87ca53, 0x3246d381, 0x48381cb6, 0x94f90564, 0x47a6ee16, 0x9b67f7c4, 0xb35a7079, 0x6f9b69ab, 0xbcc482d9, 0x60059b0b, 0x1a7b543c, - 0xc6ba4dee, 0x15e5a69c, 0xc924bf4e}, - {0x00000000, 0x87acd801, 0x0e59b103, 0x89f56902, 0x1cb26207, 0x9b1eba06, 0x12ebd304, 0x95470b05, 0x3864c50e, 0xbfc81d0f, 0x363d740d, - 0xb191ac0c, 0x24d6a709, 0xa37a7f08, 0x2a8f160a, 0xad23ce0b, 0x70c88a1d, 0xf764521c, 0x7e913b1e, 0xf93de31f, 0x6c7ae81a, 0xebd6301b, - 0x62235919, 0xe58f8118, 0x48ac4f13, 0xcf009712, 0x46f5fe10, 0xc1592611, 0x541e2d14, 0xd3b2f515, 0x5a479c17, 0xddeb4416, 0xe090153b, - 0x673ccd3a, 0xeec9a438, 0x69657c39, 0xfc22773c, 0x7b8eaf3d, 0xf27bc63f, 0x75d71e3e, 0xd8f4d035, 0x5f580834, 0xd6ad6136, 0x5101b937, - 0xc446b232, 0x43ea6a33, 0xca1f0331, 0x4db3db30, 0x90589f26, 0x17f44727, 0x9e012e25, 0x19adf624, 0x8ceafd21, 0x0b462520, 0x82b34c22, - 0x051f9423, 0xa83c5a28, 0x2f908229, 0xa665eb2b, 0x21c9332a, 0xb48e382f, 0x3322e02e, 0xbad7892c, 0x3d7b512d, 0xc0212b76, 0x478df377, - 0xce789a75, 0x49d44274, 0xdc934971, 0x5b3f9170, 0xd2caf872, 0x55662073, 0xf845ee78, 0x7fe93679, 0xf61c5f7b, 0x71b0877a, 0xe4f78c7f, - 0x635b547e, 0xeaae3d7c, 0x6d02e57d, 0xb0e9a16b, 0x3745796a, 0xbeb01068, 0x391cc869, 0xac5bc36c, 0x2bf71b6d, 0xa202726f, 0x25aeaa6e, - 0x888d6465, 0x0f21bc64, 0x86d4d566, 0x01780d67, 0x943f0662, 0x1393de63, 0x9a66b761, 0x1dca6f60, 0x20b13e4d, 0xa71de64c, 0x2ee88f4e, - 0xa944574f, 0x3c035c4a, 0xbbaf844b, 0x325aed49, 0xb5f63548, 0x18d5fb43, 0x9f792342, 0x168c4a40, 0x91209241, 0x04679944, 0x83cb4145, - 0x0a3e2847, 0x8d92f046, 0x5079b450, 0xd7d56c51, 0x5e200553, 0xd98cdd52, 0x4ccbd657, 0xcb670e56, 0x42926754, 0xc53ebf55, 0x681d715e, - 0xefb1a95f, 0x6644c05d, 0xe1e8185c, 0x74af1359, 0xf303cb58, 0x7af6a25a, 0xfd5a7a5b, 0x804356ec, 0x07ef8eed, 0x8e1ae7ef, 0x09b63fee, - 0x9cf134eb, 0x1b5decea, 0x92a885e8, 0x15045de9, 0xb82793e2, 0x3f8b4be3, 0xb67e22e1, 0x31d2fae0, 0xa495f1e5, 0x233929e4, 0xaacc40e6, - 0x2d6098e7, 0xf08bdcf1, 0x772704f0, 0xfed26df2, 0x797eb5f3, 0xec39bef6, 0x6b9566f7, 0xe2600ff5, 0x65ccd7f4, 0xc8ef19ff, 0x4f43c1fe, - 0xc6b6a8fc, 0x411a70fd, 0xd45d7bf8, 0x53f1a3f9, 0xda04cafb, 0x5da812fa, 0x60d343d7, 0xe77f9bd6, 0x6e8af2d4, 0xe9262ad5, 0x7c6121d0, - 0xfbcdf9d1, 0x723890d3, 0xf59448d2, 0x58b786d9, 0xdf1b5ed8, 0x56ee37da, 0xd142efdb, 0x4405e4de, 0xc3a93cdf, 0x4a5c55dd, 0xcdf08ddc, - 0x101bc9ca, 0x97b711cb, 0x1e4278c9, 0x99eea0c8, 0x0ca9abcd, 0x8b0573cc, 0x02f01ace, 0x855cc2cf, 0x287f0cc4, 0xafd3d4c5, 0x2626bdc7, - 0xa18a65c6, 0x34cd6ec3, 0xb361b6c2, 0x3a94dfc0, 0xbd3807c1, 0x40627d9a, 0xc7cea59b, 0x4e3bcc99, 0xc9971498, 0x5cd01f9d, 0xdb7cc79c, - 0x5289ae9e, 0xd525769f, 0x7806b894, 0xffaa6095, 0x765f0997, 0xf1f3d196, 0x64b4da93, 0xe3180292, 0x6aed6b90, 0xed41b391, 0x30aaf787, - 0xb7062f86, 0x3ef34684, 0xb95f9e85, 0x2c189580, 0xabb44d81, 0x22412483, 0xa5edfc82, 0x08ce3289, 0x8f62ea88, 0x0697838a, 0x813b5b8b, - 0x147c508e, 0x93d0888f, 0x1a25e18d, 0x9d89398c, 0xa0f268a1, 0x275eb0a0, 0xaeabd9a2, 0x290701a3, 0xbc400aa6, 0x3becd2a7, 0xb219bba5, - 0x35b563a4, 0x9896adaf, 0x1f3a75ae, 0x96cf1cac, 0x1163c4ad, 0x8424cfa8, 0x038817a9, 0x8a7d7eab, 0x0dd1a6aa, 0xd03ae2bc, 0x57963abd, - 0xde6353bf, 0x59cf8bbe, 0xcc8880bb, 0x4b2458ba, 0xc2d131b8, 0x457de9b9, 0xe85e27b2, 0x6ff2ffb3, 0xe60796b1, 0x61ab4eb0, 0xf4ec45b5, - 0x73409db4, 0xfab5f4b6, 0x7d192cb7}, - {0x00000000, 0xb79a6ddc, 0xd9281abc, 0x6eb27760, 0x054cf57c, 0xb2d698a0, 0xdc64efc0, 0x6bfe821c, 0x0a98eaf9, 0xbd028725, 0xd3b0f045, - 0x642a9d99, 0x0fd41f85, 0xb84e7259, 0xd6fc0539, 0x616668e5, 0xa32d14f7, 0x14b7792b, 0x7a050e4b, 0xcd9f6397, 0xa661e18b, 0x11fb8c57, - 0x7f49fb37, 0xc8d396eb, 0xa9b5fe0e, 0x1e2f93d2, 0x709de4b2, 0xc707896e, 0xacf90b72, 0x1b6366ae, 0x75d111ce, 0xc24b7c12, 0xf146e9ea, - 0x46dc8436, 0x286ef356, 0x9ff49e8a, 0xf40a1c96, 0x4390714a, 0x2d22062a, 0x9ab86bf6, 0xfbde0313, 0x4c446ecf, 0x22f619af, 0x956c7473, - 0xfe92f66f, 0x49089bb3, 0x27baecd3, 0x9020810f, 0x526bfd1d, 0xe5f190c1, 0x8b43e7a1, 0x3cd98a7d, 0x57270861, 0xe0bd65bd, 0x8e0f12dd, - 0x39957f01, 0x58f317e4, 0xef697a38, 0x81db0d58, 0x36416084, 0x5dbfe298, 0xea258f44, 0x8497f824, 0x330d95f8, 0x559013d1, 0xe20a7e0d, - 0x8cb8096d, 0x3b2264b1, 0x50dce6ad, 0xe7468b71, 0x89f4fc11, 0x3e6e91cd, 0x5f08f928, 0xe89294f4, 0x8620e394, 0x31ba8e48, 0x5a440c54, - 0xedde6188, 0x836c16e8, 0x34f67b34, 0xf6bd0726, 0x41276afa, 0x2f951d9a, 0x980f7046, 0xf3f1f25a, 0x446b9f86, 0x2ad9e8e6, 0x9d43853a, - 0xfc25eddf, 0x4bbf8003, 0x250df763, 0x92979abf, 0xf96918a3, 0x4ef3757f, 0x2041021f, 0x97db6fc3, 0xa4d6fa3b, 0x134c97e7, 0x7dfee087, - 0xca648d5b, 0xa19a0f47, 0x1600629b, 0x78b215fb, 0xcf287827, 0xae4e10c2, 0x19d47d1e, 0x77660a7e, 0xc0fc67a2, 0xab02e5be, 0x1c988862, - 0x722aff02, 0xc5b092de, 0x07fbeecc, 0xb0618310, 0xded3f470, 0x694999ac, 0x02b71bb0, 0xb52d766c, 0xdb9f010c, 0x6c056cd0, 0x0d630435, - 0xbaf969e9, 0xd44b1e89, 0x63d17355, 0x082ff149, 0xbfb59c95, 0xd107ebf5, 0x669d8629, 0x1d3de6a6, 0xaaa78b7a, 0xc415fc1a, 0x738f91c6, - 0x187113da, 0xafeb7e06, 0xc1590966, 0x76c364ba, 0x17a50c5f, 0xa03f6183, 0xce8d16e3, 0x79177b3f, 0x12e9f923, 0xa57394ff, 0xcbc1e39f, - 0x7c5b8e43, 0xbe10f251, 0x098a9f8d, 0x6738e8ed, 0xd0a28531, 0xbb5c072d, 0x0cc66af1, 0x62741d91, 0xd5ee704d, 0xb48818a8, 0x03127574, - 0x6da00214, 0xda3a6fc8, 0xb1c4edd4, 0x065e8008, 0x68ecf768, 0xdf769ab4, 0xec7b0f4c, 0x5be16290, 0x355315f0, 0x82c9782c, 0xe937fa30, - 0x5ead97ec, 0x301fe08c, 0x87858d50, 0xe6e3e5b5, 0x51798869, 0x3fcbff09, 0x885192d5, 0xe3af10c9, 0x54357d15, 0x3a870a75, 0x8d1d67a9, - 0x4f561bbb, 0xf8cc7667, 0x967e0107, 0x21e46cdb, 0x4a1aeec7, 0xfd80831b, 0x9332f47b, 0x24a899a7, 0x45cef142, 0xf2549c9e, 0x9ce6ebfe, - 0x2b7c8622, 0x4082043e, 0xf71869e2, 0x99aa1e82, 0x2e30735e, 0x48adf577, 0xff3798ab, 0x9185efcb, 0x261f8217, 0x4de1000b, 0xfa7b6dd7, - 0x94c91ab7, 0x2353776b, 0x42351f8e, 0xf5af7252, 0x9b1d0532, 0x2c8768ee, 0x4779eaf2, 0xf0e3872e, 0x9e51f04e, 0x29cb9d92, 0xeb80e180, - 0x5c1a8c5c, 0x32a8fb3c, 0x853296e0, 0xeecc14fc, 0x59567920, 0x37e40e40, 0x807e639c, 0xe1180b79, 0x568266a5, 0x383011c5, 0x8faa7c19, - 0xe454fe05, 0x53ce93d9, 0x3d7ce4b9, 0x8ae68965, 0xb9eb1c9d, 0x0e717141, 0x60c30621, 0xd7596bfd, 0xbca7e9e1, 0x0b3d843d, 0x658ff35d, - 0xd2159e81, 0xb373f664, 0x04e99bb8, 0x6a5becd8, 0xddc18104, 0xb63f0318, 0x01a56ec4, 0x6f1719a4, 0xd88d7478, 0x1ac6086a, 0xad5c65b6, - 0xc3ee12d6, 0x74747f0a, 0x1f8afd16, 0xa81090ca, 0xc6a2e7aa, 0x71388a76, 0x105ee293, 0xa7c48f4f, 0xc976f82f, 0x7eec95f3, 0x151217ef, - 0xa2887a33, 0xcc3a0d53, 0x7ba0608f}, - {0x00000000, 0x8d670d49, 0x1acf1a92, 0x97a817db, 0x8383f420, 0x0ee4f969, 0x994ceeb2, 0x142be3fb, 0x0607e941, 0x8b60e408, 0x1cc8f3d3, - 0x91affe9a, 0x85841d61, 0x08e31028, 0x9f4b07f3, 0x122c0aba, 0x0c0ed283, 0x8169dfca, 0x16c1c811, 0x9ba6c558, 0x8f8d26a3, 0x02ea2bea, - 0x95423c31, 0x18253178, 0x0a093bc2, 0x876e368b, 0x10c62150, 0x9da12c19, 0x898acfe2, 0x04edc2ab, 0x9345d570, 0x1e22d839, 0xaf016503, - 0x2266684a, 0xb5ce7f91, 0x38a972d8, 0x2c829123, 0xa1e59c6a, 0x364d8bb1, 0xbb2a86f8, 0xa9068c42, 0x2461810b, 0xb3c996d0, 0x3eae9b99, - 0x2a857862, 0xa7e2752b, 0x304a62f0, 0xbd2d6fb9, 0xa30fb780, 0x2e68bac9, 0xb9c0ad12, 0x34a7a05b, 0x208c43a0, 0xadeb4ee9, 0x3a435932, - 0xb724547b, 0xa5085ec1, 0x286f5388, 0xbfc74453, 0x32a0491a, 0x268baae1, 0xabeca7a8, 0x3c44b073, 0xb123bd3a, 0x5e03ca06, 0xd364c74f, - 0x44ccd094, 0xc9abdddd, 0xdd803e26, 0x50e7336f, 0xc74f24b4, 0x4a2829fd, 0x58042347, 0xd5632e0e, 0x42cb39d5, 0xcfac349c, 0xdb87d767, - 0x56e0da2e, 0xc148cdf5, 0x4c2fc0bc, 0x520d1885, 0xdf6a15cc, 0x48c20217, 0xc5a50f5e, 0xd18eeca5, 0x5ce9e1ec, 0xcb41f637, 0x4626fb7e, - 0x540af1c4, 0xd96dfc8d, 0x4ec5eb56, 0xc3a2e61f, 0xd78905e4, 0x5aee08ad, 0xcd461f76, 0x4021123f, 0xf102af05, 0x7c65a24c, 0xebcdb597, - 0x66aab8de, 0x72815b25, 0xffe6566c, 0x684e41b7, 0xe5294cfe, 0xf7054644, 0x7a624b0d, 0xedca5cd6, 0x60ad519f, 0x7486b264, 0xf9e1bf2d, - 0x6e49a8f6, 0xe32ea5bf, 0xfd0c7d86, 0x706b70cf, 0xe7c36714, 0x6aa46a5d, 0x7e8f89a6, 0xf3e884ef, 0x64409334, 0xe9279e7d, 0xfb0b94c7, - 0x766c998e, 0xe1c48e55, 0x6ca3831c, 0x788860e7, 0xf5ef6dae, 0x62477a75, 0xef20773c, 0xbc06940d, 0x31619944, 0xa6c98e9f, 0x2bae83d6, - 0x3f85602d, 0xb2e26d64, 0x254a7abf, 0xa82d77f6, 0xba017d4c, 0x37667005, 0xa0ce67de, 0x2da96a97, 0x3982896c, 0xb4e58425, 0x234d93fe, - 0xae2a9eb7, 0xb008468e, 0x3d6f4bc7, 0xaac75c1c, 0x27a05155, 0x338bb2ae, 0xbeecbfe7, 0x2944a83c, 0xa423a575, 0xb60fafcf, 0x3b68a286, - 0xacc0b55d, 0x21a7b814, 0x358c5bef, 0xb8eb56a6, 0x2f43417d, 0xa2244c34, 0x1307f10e, 0x9e60fc47, 0x09c8eb9c, 0x84afe6d5, 0x9084052e, - 0x1de30867, 0x8a4b1fbc, 0x072c12f5, 0x1500184f, 0x98671506, 0x0fcf02dd, 0x82a80f94, 0x9683ec6f, 0x1be4e126, 0x8c4cf6fd, 0x012bfbb4, - 0x1f09238d, 0x926e2ec4, 0x05c6391f, 0x88a13456, 0x9c8ad7ad, 0x11eddae4, 0x8645cd3f, 0x0b22c076, 0x190ecacc, 0x9469c785, 0x03c1d05e, - 0x8ea6dd17, 0x9a8d3eec, 0x17ea33a5, 0x8042247e, 0x0d252937, 0xe2055e0b, 0x6f625342, 0xf8ca4499, 0x75ad49d0, 0x6186aa2b, 0xece1a762, - 0x7b49b0b9, 0xf62ebdf0, 0xe402b74a, 0x6965ba03, 0xfecdadd8, 0x73aaa091, 0x6781436a, 0xeae64e23, 0x7d4e59f8, 0xf02954b1, 0xee0b8c88, - 0x636c81c1, 0xf4c4961a, 0x79a39b53, 0x6d8878a8, 0xe0ef75e1, 0x7747623a, 0xfa206f73, 0xe80c65c9, 0x656b6880, 0xf2c37f5b, 0x7fa47212, - 0x6b8f91e9, 0xe6e89ca0, 0x71408b7b, 0xfc278632, 0x4d043b08, 0xc0633641, 0x57cb219a, 0xdaac2cd3, 0xce87cf28, 0x43e0c261, 0xd448d5ba, - 0x592fd8f3, 0x4b03d249, 0xc664df00, 0x51ccc8db, 0xdcabc592, 0xc8802669, 0x45e72b20, 0xd24f3cfb, 0x5f2831b2, 0x410ae98b, 0xcc6de4c2, - 0x5bc5f319, 0xd6a2fe50, 0xc2891dab, 0x4fee10e2, 0xd8460739, 0x55210a70, 0x470d00ca, 0xca6a0d83, 0x5dc21a58, 0xd0a51711, 0xc48ef4ea, - 0x49e9f9a3, 0xde41ee78, 0x5326e331}, - {0x00000000, 0x780d281b, 0xf01a5036, 0x8817782d, 0xe035a06c, 0x98388877, 0x102ff05a, 0x6822d841, 0xc06b40d9, 0xb86668c2, 0x307110ef, - 0x487c38f4, 0x205ee0b5, 0x5853c8ae, 0xd044b083, 0xa8499898, 0x37ca41b6, 0x4fc769ad, 0xc7d01180, 0xbfdd399b, 0xd7ffe1da, 0xaff2c9c1, - 0x27e5b1ec, 0x5fe899f7, 0xf7a1016f, 0x8fac2974, 0x07bb5159, 0x7fb67942, 0x1794a103, 0x6f998918, 0xe78ef135, 0x9f83d92e, 0xd9894268, - 0xa1846a73, 0x2993125e, 0x519e3a45, 0x39bce204, 0x41b1ca1f, 0xc9a6b232, 0xb1ab9a29, 0x19e202b1, 0x61ef2aaa, 0xe9f85287, 0x91f57a9c, - 0xf9d7a2dd, 0x81da8ac6, 0x09cdf2eb, 0x71c0daf0, 0xee4303de, 0x964e2bc5, 0x1e5953e8, 0x66547bf3, 0x0e76a3b2, 0x767b8ba9, 0xfe6cf384, - 0x8661db9f, 0x2e284307, 0x56256b1c, 0xde321331, 0xa63f3b2a, 0xce1de36b, 0xb610cb70, 0x3e07b35d, 0x460a9b46, 0xb21385d0, 0xca1eadcb, - 0x4209d5e6, 0x3a04fdfd, 0x522625bc, 0x2a2b0da7, 0xa23c758a, 0xda315d91, 0x7278c509, 0x0a75ed12, 0x8262953f, 0xfa6fbd24, 0x924d6565, - 0xea404d7e, 0x62573553, 0x1a5a1d48, 0x85d9c466, 0xfdd4ec7d, 0x75c39450, 0x0dcebc4b, 0x65ec640a, 0x1de14c11, 0x95f6343c, 0xedfb1c27, - 0x45b284bf, 0x3dbfaca4, 0xb5a8d489, 0xcda5fc92, 0xa58724d3, 0xdd8a0cc8, 0x559d74e5, 0x2d905cfe, 0x6b9ac7b8, 0x1397efa3, 0x9b80978e, - 0xe38dbf95, 0x8baf67d4, 0xf3a24fcf, 0x7bb537e2, 0x03b81ff9, 0xabf18761, 0xd3fcaf7a, 0x5bebd757, 0x23e6ff4c, 0x4bc4270d, 0x33c90f16, - 0xbbde773b, 0xc3d35f20, 0x5c50860e, 0x245dae15, 0xac4ad638, 0xd447fe23, 0xbc652662, 0xc4680e79, 0x4c7f7654, 0x34725e4f, 0x9c3bc6d7, - 0xe436eecc, 0x6c2196e1, 0x142cbefa, 0x7c0e66bb, 0x04034ea0, 0x8c14368d, 0xf4191e96, 0xd33acba5, 0xab37e3be, 0x23209b93, 0x5b2db388, - 0x330f6bc9, 0x4b0243d2, 0xc3153bff, 0xbb1813e4, 0x13518b7c, 0x6b5ca367, 0xe34bdb4a, 0x9b46f351, 0xf3642b10, 0x8b69030b, 0x037e7b26, - 0x7b73533d, 0xe4f08a13, 0x9cfda208, 0x14eada25, 0x6ce7f23e, 0x04c52a7f, 0x7cc80264, 0xf4df7a49, 0x8cd25252, 0x249bcaca, 0x5c96e2d1, - 0xd4819afc, 0xac8cb2e7, 0xc4ae6aa6, 0xbca342bd, 0x34b43a90, 0x4cb9128b, 0x0ab389cd, 0x72bea1d6, 0xfaa9d9fb, 0x82a4f1e0, 0xea8629a1, - 0x928b01ba, 0x1a9c7997, 0x6291518c, 0xcad8c914, 0xb2d5e10f, 0x3ac29922, 0x42cfb139, 0x2aed6978, 0x52e04163, 0xdaf7394e, 0xa2fa1155, - 0x3d79c87b, 0x4574e060, 0xcd63984d, 0xb56eb056, 0xdd4c6817, 0xa541400c, 0x2d563821, 0x555b103a, 0xfd1288a2, 0x851fa0b9, 0x0d08d894, - 0x7505f08f, 0x1d2728ce, 0x652a00d5, 0xed3d78f8, 0x953050e3, 0x61294e75, 0x1924666e, 0x91331e43, 0xe93e3658, 0x811cee19, 0xf911c602, - 0x7106be2f, 0x090b9634, 0xa1420eac, 0xd94f26b7, 0x51585e9a, 0x29557681, 0x4177aec0, 0x397a86db, 0xb16dfef6, 0xc960d6ed, 0x56e30fc3, - 0x2eee27d8, 0xa6f95ff5, 0xdef477ee, 0xb6d6afaf, 0xcedb87b4, 0x46ccff99, 0x3ec1d782, 0x96884f1a, 0xee856701, 0x66921f2c, 0x1e9f3737, - 0x76bdef76, 0x0eb0c76d, 0x86a7bf40, 0xfeaa975b, 0xb8a00c1d, 0xc0ad2406, 0x48ba5c2b, 0x30b77430, 0x5895ac71, 0x2098846a, 0xa88ffc47, - 0xd082d45c, 0x78cb4cc4, 0x00c664df, 0x88d11cf2, 0xf0dc34e9, 0x98feeca8, 0xe0f3c4b3, 0x68e4bc9e, 0x10e99485, 0x8f6a4dab, 0xf76765b0, - 0x7f701d9d, 0x077d3586, 0x6f5fedc7, 0x1752c5dc, 0x9f45bdf1, 0xe74895ea, 0x4f010d72, 0x370c2569, 0xbf1b5d44, 0xc716755f, 0xaf34ad1e, - 0xd7398505, 0x5f2efd28, 0x2723d533}, - {0x00000000, 0x1168574f, 0x22d0ae9e, 0x33b8f9d1, 0xf3bd9c39, 0xe2d5cb76, 0xd16d32a7, 0xc00565e8, 0xe67b3973, 0xf7136e3c, 0xc4ab97ed, - 0xd5c3c0a2, 0x15c6a54a, 0x04aef205, 0x37160bd4, 0x267e5c9b, 0xccf772e6, 0xdd9f25a9, 0xee27dc78, 0xff4f8b37, 0x3f4aeedf, 0x2e22b990, - 0x1d9a4041, 0x0cf2170e, 0x2a8c4b95, 0x3be41cda, 0x085ce50b, 0x1934b244, 0xd931d7ac, 0xc85980e3, 0xfbe17932, 0xea892e7d, 0x2ff224c8, - 0x3e9a7387, 0x0d228a56, 0x1c4add19, 0xdc4fb8f1, 0xcd27efbe, 0xfe9f166f, 0xeff74120, 0xc9891dbb, 0xd8e14af4, 0xeb59b325, 0xfa31e46a, - 0x3a348182, 0x2b5cd6cd, 0x18e42f1c, 0x098c7853, 0xe305562e, 0xf26d0161, 0xc1d5f8b0, 0xd0bdafff, 0x10b8ca17, 0x01d09d58, 0x32686489, - 0x230033c6, 0x057e6f5d, 0x14163812, 0x27aec1c3, 0x36c6968c, 0xf6c3f364, 0xe7aba42b, 0xd4135dfa, 0xc57b0ab5, 0xe9f98894, 0xf891dfdb, - 0xcb29260a, 0xda417145, 0x1a4414ad, 0x0b2c43e2, 0x3894ba33, 0x29fced7c, 0x0f82b1e7, 0x1eeae6a8, 0x2d521f79, 0x3c3a4836, 0xfc3f2dde, - 0xed577a91, 0xdeef8340, 0xcf87d40f, 0x250efa72, 0x3466ad3d, 0x07de54ec, 0x16b603a3, 0xd6b3664b, 0xc7db3104, 0xf463c8d5, 0xe50b9f9a, - 0xc375c301, 0xd21d944e, 0xe1a56d9f, 0xf0cd3ad0, 0x30c85f38, 0x21a00877, 0x1218f1a6, 0x0370a6e9, 0xc60bac5c, 0xd763fb13, 0xe4db02c2, - 0xf5b3558d, 0x35b63065, 0x24de672a, 0x17669efb, 0x060ec9b4, 0x2070952f, 0x3118c260, 0x02a03bb1, 0x13c86cfe, 0xd3cd0916, 0xc2a55e59, - 0xf11da788, 0xe075f0c7, 0x0afcdeba, 0x1b9489f5, 0x282c7024, 0x3944276b, 0xf9414283, 0xe82915cc, 0xdb91ec1d, 0xcaf9bb52, 0xec87e7c9, - 0xfdefb086, 0xce574957, 0xdf3f1e18, 0x1f3a7bf0, 0x0e522cbf, 0x3dead56e, 0x2c828221, 0x65eed02d, 0x74868762, 0x473e7eb3, 0x565629fc, - 0x96534c14, 0x873b1b5b, 0xb483e28a, 0xa5ebb5c5, 0x8395e95e, 0x92fdbe11, 0xa14547c0, 0xb02d108f, 0x70287567, 0x61402228, 0x52f8dbf9, - 0x43908cb6, 0xa919a2cb, 0xb871f584, 0x8bc90c55, 0x9aa15b1a, 0x5aa43ef2, 0x4bcc69bd, 0x7874906c, 0x691cc723, 0x4f629bb8, 0x5e0accf7, - 0x6db23526, 0x7cda6269, 0xbcdf0781, 0xadb750ce, 0x9e0fa91f, 0x8f67fe50, 0x4a1cf4e5, 0x5b74a3aa, 0x68cc5a7b, 0x79a40d34, 0xb9a168dc, - 0xa8c93f93, 0x9b71c642, 0x8a19910d, 0xac67cd96, 0xbd0f9ad9, 0x8eb76308, 0x9fdf3447, 0x5fda51af, 0x4eb206e0, 0x7d0aff31, 0x6c62a87e, - 0x86eb8603, 0x9783d14c, 0xa43b289d, 0xb5537fd2, 0x75561a3a, 0x643e4d75, 0x5786b4a4, 0x46eee3eb, 0x6090bf70, 0x71f8e83f, 0x424011ee, - 0x532846a1, 0x932d2349, 0x82457406, 0xb1fd8dd7, 0xa095da98, 0x8c1758b9, 0x9d7f0ff6, 0xaec7f627, 0xbfafa168, 0x7faac480, 0x6ec293cf, - 0x5d7a6a1e, 0x4c123d51, 0x6a6c61ca, 0x7b043685, 0x48bccf54, 0x59d4981b, 0x99d1fdf3, 0x88b9aabc, 0xbb01536d, 0xaa690422, 0x40e02a5f, - 0x51887d10, 0x623084c1, 0x7358d38e, 0xb35db666, 0xa235e129, 0x918d18f8, 0x80e54fb7, 0xa69b132c, 0xb7f34463, 0x844bbdb2, 0x9523eafd, - 0x55268f15, 0x444ed85a, 0x77f6218b, 0x669e76c4, 0xa3e57c71, 0xb28d2b3e, 0x8135d2ef, 0x905d85a0, 0x5058e048, 0x4130b707, 0x72884ed6, - 0x63e01999, 0x459e4502, 0x54f6124d, 0x674eeb9c, 0x7626bcd3, 0xb623d93b, 0xa74b8e74, 0x94f377a5, 0x859b20ea, 0x6f120e97, 0x7e7a59d8, - 0x4dc2a009, 0x5caaf746, 0x9caf92ae, 0x8dc7c5e1, 0xbe7f3c30, 0xaf176b7f, 0x896937e4, 0x980160ab, 0xabb9997a, 0xbad1ce35, 0x7ad4abdd, - 0x6bbcfc92, 0x58040543, 0x496c520c}, - {0x00000000, 0xcadca15b, 0x94b943b7, 0x5e65e2ec, 0x9f6e466a, 0x55b2e731, 0x0bd705dd, 0xc10ba486, 0x3edd8cd4, 0xf4012d8f, 0xaa64cf63, - 0x60b86e38, 0xa1b3cabe, 0x6b6f6be5, 0x350a8909, 0xffd62852, 0xcba7d8ad, 0x017b79f6, 0x5f1e9b1a, 0x95c23a41, 0x54c99ec7, 0x9e153f9c, - 0xc070dd70, 0x0aac7c2b, 0xf57a5479, 0x3fa6f522, 0x61c317ce, 0xab1fb695, 0x6a141213, 0xa0c8b348, 0xfead51a4, 0x3471f0ff, 0x2152705f, - 0xeb8ed104, 0xb5eb33e8, 0x7f3792b3, 0xbe3c3635, 0x74e0976e, 0x2a857582, 0xe059d4d9, 0x1f8ffc8b, 0xd5535dd0, 0x8b36bf3c, 0x41ea1e67, - 0x80e1bae1, 0x4a3d1bba, 0x1458f956, 0xde84580d, 0xeaf5a8f2, 0x202909a9, 0x7e4ceb45, 0xb4904a1e, 0x759bee98, 0xbf474fc3, 0xe122ad2f, - 0x2bfe0c74, 0xd4282426, 0x1ef4857d, 0x40916791, 0x8a4dc6ca, 0x4b46624c, 0x819ac317, 0xdfff21fb, 0x152380a0, 0x42a4e0be, 0x887841e5, - 0xd61da309, 0x1cc10252, 0xddcaa6d4, 0x1716078f, 0x4973e563, 0x83af4438, 0x7c796c6a, 0xb6a5cd31, 0xe8c02fdd, 0x221c8e86, 0xe3172a00, - 0x29cb8b5b, 0x77ae69b7, 0xbd72c8ec, 0x89033813, 0x43df9948, 0x1dba7ba4, 0xd766daff, 0x166d7e79, 0xdcb1df22, 0x82d43dce, 0x48089c95, - 0xb7deb4c7, 0x7d02159c, 0x2367f770, 0xe9bb562b, 0x28b0f2ad, 0xe26c53f6, 0xbc09b11a, 0x76d51041, 0x63f690e1, 0xa92a31ba, 0xf74fd356, - 0x3d93720d, 0xfc98d68b, 0x364477d0, 0x6821953c, 0xa2fd3467, 0x5d2b1c35, 0x97f7bd6e, 0xc9925f82, 0x034efed9, 0xc2455a5f, 0x0899fb04, - 0x56fc19e8, 0x9c20b8b3, 0xa851484c, 0x628de917, 0x3ce80bfb, 0xf634aaa0, 0x373f0e26, 0xfde3af7d, 0xa3864d91, 0x695aecca, 0x968cc498, - 0x5c5065c3, 0x0235872f, 0xc8e92674, 0x09e282f2, 0xc33e23a9, 0x9d5bc145, 0x5787601e, 0x33550079, 0xf989a122, 0xa7ec43ce, 0x6d30e295, - 0xac3b4613, 0x66e7e748, 0x388205a4, 0xf25ea4ff, 0x0d888cad, 0xc7542df6, 0x9931cf1a, 0x53ed6e41, 0x92e6cac7, 0x583a6b9c, 0x065f8970, - 0xcc83282b, 0xf8f2d8d4, 0x322e798f, 0x6c4b9b63, 0xa6973a38, 0x679c9ebe, 0xad403fe5, 0xf325dd09, 0x39f97c52, 0xc62f5400, 0x0cf3f55b, - 0x529617b7, 0x984ab6ec, 0x5941126a, 0x939db331, 0xcdf851dd, 0x0724f086, 0x12077026, 0xd8dbd17d, 0x86be3391, 0x4c6292ca, 0x8d69364c, - 0x47b59717, 0x19d075fb, 0xd30cd4a0, 0x2cdafcf2, 0xe6065da9, 0xb863bf45, 0x72bf1e1e, 0xb3b4ba98, 0x79681bc3, 0x270df92f, 0xedd15874, - 0xd9a0a88b, 0x137c09d0, 0x4d19eb3c, 0x87c54a67, 0x46ceeee1, 0x8c124fba, 0xd277ad56, 0x18ab0c0d, 0xe77d245f, 0x2da18504, 0x73c467e8, - 0xb918c6b3, 0x78136235, 0xb2cfc36e, 0xecaa2182, 0x267680d9, 0x71f1e0c7, 0xbb2d419c, 0xe548a370, 0x2f94022b, 0xee9fa6ad, 0x244307f6, - 0x7a26e51a, 0xb0fa4441, 0x4f2c6c13, 0x85f0cd48, 0xdb952fa4, 0x11498eff, 0xd0422a79, 0x1a9e8b22, 0x44fb69ce, 0x8e27c895, 0xba56386a, - 0x708a9931, 0x2eef7bdd, 0xe433da86, 0x25387e00, 0xefe4df5b, 0xb1813db7, 0x7b5d9cec, 0x848bb4be, 0x4e5715e5, 0x1032f709, 0xdaee5652, - 0x1be5f2d4, 0xd139538f, 0x8f5cb163, 0x45801038, 0x50a39098, 0x9a7f31c3, 0xc41ad32f, 0x0ec67274, 0xcfcdd6f2, 0x051177a9, 0x5b749545, - 0x91a8341e, 0x6e7e1c4c, 0xa4a2bd17, 0xfac75ffb, 0x301bfea0, 0xf1105a26, 0x3bccfb7d, 0x65a91991, 0xaf75b8ca, 0x9b044835, 0x51d8e96e, - 0x0fbd0b82, 0xc561aad9, 0x046a0e5f, 0xceb6af04, 0x90d34de8, 0x5a0fecb3, 0xa5d9c4e1, 0x6f0565ba, 0x31608756, 0xfbbc260d, 0x3ab7828b, - 0xf06b23d0, 0xae0ec13c, 0x64d26067}}; - -static const uint32_t CRCTablesSB8[8][256] = { - {0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, - 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, - 0xf4d4b551, 0x83d385c7, 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, - 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, - 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, - 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, - 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, - 0x086d3d2d, 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, - 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, - 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, - 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, - 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, - 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, - 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, - 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, - 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, - 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, - 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, - 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, - 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, - 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 0xbdbdf21c, 0xcabac28a, - 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, - 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d}, - {0x00000000, 0x191b3141, 0x32366282, 0x2b2d53c3, 0x646cc504, 0x7d77f445, 0x565aa786, 0x4f4196c7, 0xc8d98a08, 0xd1c2bb49, 0xfaefe88a, - 0xe3f4d9cb, 0xacb54f0c, 0xb5ae7e4d, 0x9e832d8e, 0x87981ccf, 0x4ac21251, 0x53d92310, 0x78f470d3, 0x61ef4192, 0x2eaed755, 0x37b5e614, - 0x1c98b5d7, 0x05838496, 0x821b9859, 0x9b00a918, 0xb02dfadb, 0xa936cb9a, 0xe6775d5d, 0xff6c6c1c, 0xd4413fdf, 0xcd5a0e9e, 0x958424a2, - 0x8c9f15e3, 0xa7b24620, 0xbea97761, 0xf1e8e1a6, 0xe8f3d0e7, 0xc3de8324, 0xdac5b265, 0x5d5daeaa, 0x44469feb, 0x6f6bcc28, 0x7670fd69, - 0x39316bae, 0x202a5aef, 0x0b07092c, 0x121c386d, 0xdf4636f3, 0xc65d07b2, 0xed705471, 0xf46b6530, 0xbb2af3f7, 0xa231c2b6, 0x891c9175, - 0x9007a034, 0x179fbcfb, 0x0e848dba, 0x25a9de79, 0x3cb2ef38, 0x73f379ff, 0x6ae848be, 0x41c51b7d, 0x58de2a3c, 0xf0794f05, 0xe9627e44, - 0xc24f2d87, 0xdb541cc6, 0x94158a01, 0x8d0ebb40, 0xa623e883, 0xbf38d9c2, 0x38a0c50d, 0x21bbf44c, 0x0a96a78f, 0x138d96ce, 0x5ccc0009, - 0x45d73148, 0x6efa628b, 0x77e153ca, 0xbabb5d54, 0xa3a06c15, 0x888d3fd6, 0x91960e97, 0xded79850, 0xc7cca911, 0xece1fad2, 0xf5facb93, - 0x7262d75c, 0x6b79e61d, 0x4054b5de, 0x594f849f, 0x160e1258, 0x0f152319, 0x243870da, 0x3d23419b, 0x65fd6ba7, 0x7ce65ae6, 0x57cb0925, - 0x4ed03864, 0x0191aea3, 0x188a9fe2, 0x33a7cc21, 0x2abcfd60, 0xad24e1af, 0xb43fd0ee, 0x9f12832d, 0x8609b26c, 0xc94824ab, 0xd05315ea, - 0xfb7e4629, 0xe2657768, 0x2f3f79f6, 0x362448b7, 0x1d091b74, 0x04122a35, 0x4b53bcf2, 0x52488db3, 0x7965de70, 0x607eef31, 0xe7e6f3fe, - 0xfefdc2bf, 0xd5d0917c, 0xcccba03d, 0x838a36fa, 0x9a9107bb, 0xb1bc5478, 0xa8a76539, 0x3b83984b, 0x2298a90a, 0x09b5fac9, 0x10aecb88, - 0x5fef5d4f, 0x46f46c0e, 0x6dd93fcd, 0x74c20e8c, 0xf35a1243, 0xea412302, 0xc16c70c1, 0xd8774180, 0x9736d747, 0x8e2de606, 0xa500b5c5, - 0xbc1b8484, 0x71418a1a, 0x685abb5b, 0x4377e898, 0x5a6cd9d9, 0x152d4f1e, 0x0c367e5f, 0x271b2d9c, 0x3e001cdd, 0xb9980012, 0xa0833153, - 0x8bae6290, 0x92b553d1, 0xddf4c516, 0xc4eff457, 0xefc2a794, 0xf6d996d5, 0xae07bce9, 0xb71c8da8, 0x9c31de6b, 0x852aef2a, 0xca6b79ed, - 0xd37048ac, 0xf85d1b6f, 0xe1462a2e, 0x66de36e1, 0x7fc507a0, 0x54e85463, 0x4df36522, 0x02b2f3e5, 0x1ba9c2a4, 0x30849167, 0x299fa026, - 0xe4c5aeb8, 0xfdde9ff9, 0xd6f3cc3a, 0xcfe8fd7b, 0x80a96bbc, 0x99b25afd, 0xb29f093e, 0xab84387f, 0x2c1c24b0, 0x350715f1, 0x1e2a4632, - 0x07317773, 0x4870e1b4, 0x516bd0f5, 0x7a468336, 0x635db277, 0xcbfad74e, 0xd2e1e60f, 0xf9ccb5cc, 0xe0d7848d, 0xaf96124a, 0xb68d230b, - 0x9da070c8, 0x84bb4189, 0x03235d46, 0x1a386c07, 0x31153fc4, 0x280e0e85, 0x674f9842, 0x7e54a903, 0x5579fac0, 0x4c62cb81, 0x8138c51f, - 0x9823f45e, 0xb30ea79d, 0xaa1596dc, 0xe554001b, 0xfc4f315a, 0xd7626299, 0xce7953d8, 0x49e14f17, 0x50fa7e56, 0x7bd72d95, 0x62cc1cd4, - 0x2d8d8a13, 0x3496bb52, 0x1fbbe891, 0x06a0d9d0, 0x5e7ef3ec, 0x4765c2ad, 0x6c48916e, 0x7553a02f, 0x3a1236e8, 0x230907a9, 0x0824546a, - 0x113f652b, 0x96a779e4, 0x8fbc48a5, 0xa4911b66, 0xbd8a2a27, 0xf2cbbce0, 0xebd08da1, 0xc0fdde62, 0xd9e6ef23, 0x14bce1bd, 0x0da7d0fc, - 0x268a833f, 0x3f91b27e, 0x70d024b9, 0x69cb15f8, 0x42e6463b, 0x5bfd777a, 0xdc656bb5, 0xc57e5af4, 0xee530937, 0xf7483876, 0xb809aeb1, - 0xa1129ff0, 0x8a3fcc33, 0x9324fd72}, - {0x00000000, 0x01c26a37, 0x0384d46e, 0x0246be59, 0x0709a8dc, 0x06cbc2eb, 0x048d7cb2, 0x054f1685, 0x0e1351b8, 0x0fd13b8f, 0x0d9785d6, - 0x0c55efe1, 0x091af964, 0x08d89353, 0x0a9e2d0a, 0x0b5c473d, 0x1c26a370, 0x1de4c947, 0x1fa2771e, 0x1e601d29, 0x1b2f0bac, 0x1aed619b, - 0x18abdfc2, 0x1969b5f5, 0x1235f2c8, 0x13f798ff, 0x11b126a6, 0x10734c91, 0x153c5a14, 0x14fe3023, 0x16b88e7a, 0x177ae44d, 0x384d46e0, - 0x398f2cd7, 0x3bc9928e, 0x3a0bf8b9, 0x3f44ee3c, 0x3e86840b, 0x3cc03a52, 0x3d025065, 0x365e1758, 0x379c7d6f, 0x35dac336, 0x3418a901, - 0x3157bf84, 0x3095d5b3, 0x32d36bea, 0x331101dd, 0x246be590, 0x25a98fa7, 0x27ef31fe, 0x262d5bc9, 0x23624d4c, 0x22a0277b, 0x20e69922, - 0x2124f315, 0x2a78b428, 0x2bbade1f, 0x29fc6046, 0x283e0a71, 0x2d711cf4, 0x2cb376c3, 0x2ef5c89a, 0x2f37a2ad, 0x709a8dc0, 0x7158e7f7, - 0x731e59ae, 0x72dc3399, 0x7793251c, 0x76514f2b, 0x7417f172, 0x75d59b45, 0x7e89dc78, 0x7f4bb64f, 0x7d0d0816, 0x7ccf6221, 0x798074a4, - 0x78421e93, 0x7a04a0ca, 0x7bc6cafd, 0x6cbc2eb0, 0x6d7e4487, 0x6f38fade, 0x6efa90e9, 0x6bb5866c, 0x6a77ec5b, 0x68315202, 0x69f33835, - 0x62af7f08, 0x636d153f, 0x612bab66, 0x60e9c151, 0x65a6d7d4, 0x6464bde3, 0x662203ba, 0x67e0698d, 0x48d7cb20, 0x4915a117, 0x4b531f4e, - 0x4a917579, 0x4fde63fc, 0x4e1c09cb, 0x4c5ab792, 0x4d98dda5, 0x46c49a98, 0x4706f0af, 0x45404ef6, 0x448224c1, 0x41cd3244, 0x400f5873, - 0x4249e62a, 0x438b8c1d, 0x54f16850, 0x55330267, 0x5775bc3e, 0x56b7d609, 0x53f8c08c, 0x523aaabb, 0x507c14e2, 0x51be7ed5, 0x5ae239e8, - 0x5b2053df, 0x5966ed86, 0x58a487b1, 0x5deb9134, 0x5c29fb03, 0x5e6f455a, 0x5fad2f6d, 0xe1351b80, 0xe0f771b7, 0xe2b1cfee, 0xe373a5d9, - 0xe63cb35c, 0xe7fed96b, 0xe5b86732, 0xe47a0d05, 0xef264a38, 0xeee4200f, 0xeca29e56, 0xed60f461, 0xe82fe2e4, 0xe9ed88d3, 0xebab368a, - 0xea695cbd, 0xfd13b8f0, 0xfcd1d2c7, 0xfe976c9e, 0xff5506a9, 0xfa1a102c, 0xfbd87a1b, 0xf99ec442, 0xf85cae75, 0xf300e948, 0xf2c2837f, - 0xf0843d26, 0xf1465711, 0xf4094194, 0xf5cb2ba3, 0xf78d95fa, 0xf64fffcd, 0xd9785d60, 0xd8ba3757, 0xdafc890e, 0xdb3ee339, 0xde71f5bc, - 0xdfb39f8b, 0xddf521d2, 0xdc374be5, 0xd76b0cd8, 0xd6a966ef, 0xd4efd8b6, 0xd52db281, 0xd062a404, 0xd1a0ce33, 0xd3e6706a, 0xd2241a5d, - 0xc55efe10, 0xc49c9427, 0xc6da2a7e, 0xc7184049, 0xc25756cc, 0xc3953cfb, 0xc1d382a2, 0xc011e895, 0xcb4dafa8, 0xca8fc59f, 0xc8c97bc6, - 0xc90b11f1, 0xcc440774, 0xcd866d43, 0xcfc0d31a, 0xce02b92d, 0x91af9640, 0x906dfc77, 0x922b422e, 0x93e92819, 0x96a63e9c, 0x976454ab, - 0x9522eaf2, 0x94e080c5, 0x9fbcc7f8, 0x9e7eadcf, 0x9c381396, 0x9dfa79a1, 0x98b56f24, 0x99770513, 0x9b31bb4a, 0x9af3d17d, 0x8d893530, - 0x8c4b5f07, 0x8e0de15e, 0x8fcf8b69, 0x8a809dec, 0x8b42f7db, 0x89044982, 0x88c623b5, 0x839a6488, 0x82580ebf, 0x801eb0e6, 0x81dcdad1, - 0x8493cc54, 0x8551a663, 0x8717183a, 0x86d5720d, 0xa9e2d0a0, 0xa820ba97, 0xaa6604ce, 0xaba46ef9, 0xaeeb787c, 0xaf29124b, 0xad6fac12, - 0xacadc625, 0xa7f18118, 0xa633eb2f, 0xa4755576, 0xa5b73f41, 0xa0f829c4, 0xa13a43f3, 0xa37cfdaa, 0xa2be979d, 0xb5c473d0, 0xb40619e7, - 0xb640a7be, 0xb782cd89, 0xb2cddb0c, 0xb30fb13b, 0xb1490f62, 0xb08b6555, 0xbbd72268, 0xba15485f, 0xb853f606, 0xb9919c31, 0xbcde8ab4, - 0xbd1ce083, 0xbf5a5eda, 0xbe9834ed}, - {0x00000000, 0xb8bc6765, 0xaa09c88b, 0x12b5afee, 0x8f629757, 0x37def032, 0x256b5fdc, 0x9dd738b9, 0xc5b428ef, 0x7d084f8a, 0x6fbde064, - 0xd7018701, 0x4ad6bfb8, 0xf26ad8dd, 0xe0df7733, 0x58631056, 0x5019579f, 0xe8a530fa, 0xfa109f14, 0x42acf871, 0xdf7bc0c8, 0x67c7a7ad, - 0x75720843, 0xcdce6f26, 0x95ad7f70, 0x2d111815, 0x3fa4b7fb, 0x8718d09e, 0x1acfe827, 0xa2738f42, 0xb0c620ac, 0x087a47c9, 0xa032af3e, - 0x188ec85b, 0x0a3b67b5, 0xb28700d0, 0x2f503869, 0x97ec5f0c, 0x8559f0e2, 0x3de59787, 0x658687d1, 0xdd3ae0b4, 0xcf8f4f5a, 0x7733283f, - 0xeae41086, 0x525877e3, 0x40edd80d, 0xf851bf68, 0xf02bf8a1, 0x48979fc4, 0x5a22302a, 0xe29e574f, 0x7f496ff6, 0xc7f50893, 0xd540a77d, - 0x6dfcc018, 0x359fd04e, 0x8d23b72b, 0x9f9618c5, 0x272a7fa0, 0xbafd4719, 0x0241207c, 0x10f48f92, 0xa848e8f7, 0x9b14583d, 0x23a83f58, - 0x311d90b6, 0x89a1f7d3, 0x1476cf6a, 0xaccaa80f, 0xbe7f07e1, 0x06c36084, 0x5ea070d2, 0xe61c17b7, 0xf4a9b859, 0x4c15df3c, 0xd1c2e785, - 0x697e80e0, 0x7bcb2f0e, 0xc377486b, 0xcb0d0fa2, 0x73b168c7, 0x6104c729, 0xd9b8a04c, 0x446f98f5, 0xfcd3ff90, 0xee66507e, 0x56da371b, - 0x0eb9274d, 0xb6054028, 0xa4b0efc6, 0x1c0c88a3, 0x81dbb01a, 0x3967d77f, 0x2bd27891, 0x936e1ff4, 0x3b26f703, 0x839a9066, 0x912f3f88, - 0x299358ed, 0xb4446054, 0x0cf80731, 0x1e4da8df, 0xa6f1cfba, 0xfe92dfec, 0x462eb889, 0x549b1767, 0xec277002, 0x71f048bb, 0xc94c2fde, - 0xdbf98030, 0x6345e755, 0x6b3fa09c, 0xd383c7f9, 0xc1366817, 0x798a0f72, 0xe45d37cb, 0x5ce150ae, 0x4e54ff40, 0xf6e89825, 0xae8b8873, - 0x1637ef16, 0x048240f8, 0xbc3e279d, 0x21e91f24, 0x99557841, 0x8be0d7af, 0x335cb0ca, 0xed59b63b, 0x55e5d15e, 0x47507eb0, 0xffec19d5, - 0x623b216c, 0xda874609, 0xc832e9e7, 0x708e8e82, 0x28ed9ed4, 0x9051f9b1, 0x82e4565f, 0x3a58313a, 0xa78f0983, 0x1f336ee6, 0x0d86c108, - 0xb53aa66d, 0xbd40e1a4, 0x05fc86c1, 0x1749292f, 0xaff54e4a, 0x322276f3, 0x8a9e1196, 0x982bbe78, 0x2097d91d, 0x78f4c94b, 0xc048ae2e, - 0xd2fd01c0, 0x6a4166a5, 0xf7965e1c, 0x4f2a3979, 0x5d9f9697, 0xe523f1f2, 0x4d6b1905, 0xf5d77e60, 0xe762d18e, 0x5fdeb6eb, 0xc2098e52, - 0x7ab5e937, 0x680046d9, 0xd0bc21bc, 0x88df31ea, 0x3063568f, 0x22d6f961, 0x9a6a9e04, 0x07bda6bd, 0xbf01c1d8, 0xadb46e36, 0x15080953, - 0x1d724e9a, 0xa5ce29ff, 0xb77b8611, 0x0fc7e174, 0x9210d9cd, 0x2aacbea8, 0x38191146, 0x80a57623, 0xd8c66675, 0x607a0110, 0x72cfaefe, - 0xca73c99b, 0x57a4f122, 0xef189647, 0xfdad39a9, 0x45115ecc, 0x764dee06, 0xcef18963, 0xdc44268d, 0x64f841e8, 0xf92f7951, 0x41931e34, - 0x5326b1da, 0xeb9ad6bf, 0xb3f9c6e9, 0x0b45a18c, 0x19f00e62, 0xa14c6907, 0x3c9b51be, 0x842736db, 0x96929935, 0x2e2efe50, 0x2654b999, - 0x9ee8defc, 0x8c5d7112, 0x34e11677, 0xa9362ece, 0x118a49ab, 0x033fe645, 0xbb838120, 0xe3e09176, 0x5b5cf613, 0x49e959fd, 0xf1553e98, - 0x6c820621, 0xd43e6144, 0xc68bceaa, 0x7e37a9cf, 0xd67f4138, 0x6ec3265d, 0x7c7689b3, 0xc4caeed6, 0x591dd66f, 0xe1a1b10a, 0xf3141ee4, - 0x4ba87981, 0x13cb69d7, 0xab770eb2, 0xb9c2a15c, 0x017ec639, 0x9ca9fe80, 0x241599e5, 0x36a0360b, 0x8e1c516e, 0x866616a7, 0x3eda71c2, - 0x2c6fde2c, 0x94d3b949, 0x090481f0, 0xb1b8e695, 0xa30d497b, 0x1bb12e1e, 0x43d23e48, 0xfb6e592d, 0xe9dbf6c3, 0x516791a6, 0xccb0a91f, - 0x740cce7a, 0x66b96194, 0xde0506f1}, - {0x00000000, 0x3d6029b0, 0x7ac05360, 0x47a07ad0, 0xf580a6c0, 0xc8e08f70, 0x8f40f5a0, 0xb220dc10, 0x30704bc1, 0x0d106271, 0x4ab018a1, - 0x77d03111, 0xc5f0ed01, 0xf890c4b1, 0xbf30be61, 0x825097d1, 0x60e09782, 0x5d80be32, 0x1a20c4e2, 0x2740ed52, 0x95603142, 0xa80018f2, - 0xefa06222, 0xd2c04b92, 0x5090dc43, 0x6df0f5f3, 0x2a508f23, 0x1730a693, 0xa5107a83, 0x98705333, 0xdfd029e3, 0xe2b00053, 0xc1c12f04, - 0xfca106b4, 0xbb017c64, 0x866155d4, 0x344189c4, 0x0921a074, 0x4e81daa4, 0x73e1f314, 0xf1b164c5, 0xccd14d75, 0x8b7137a5, 0xb6111e15, - 0x0431c205, 0x3951ebb5, 0x7ef19165, 0x4391b8d5, 0xa121b886, 0x9c419136, 0xdbe1ebe6, 0xe681c256, 0x54a11e46, 0x69c137f6, 0x2e614d26, - 0x13016496, 0x9151f347, 0xac31daf7, 0xeb91a027, 0xd6f18997, 0x64d15587, 0x59b17c37, 0x1e1106e7, 0x23712f57, 0x58f35849, 0x659371f9, - 0x22330b29, 0x1f532299, 0xad73fe89, 0x9013d739, 0xd7b3ade9, 0xead38459, 0x68831388, 0x55e33a38, 0x124340e8, 0x2f236958, 0x9d03b548, - 0xa0639cf8, 0xe7c3e628, 0xdaa3cf98, 0x3813cfcb, 0x0573e67b, 0x42d39cab, 0x7fb3b51b, 0xcd93690b, 0xf0f340bb, 0xb7533a6b, 0x8a3313db, - 0x0863840a, 0x3503adba, 0x72a3d76a, 0x4fc3feda, 0xfde322ca, 0xc0830b7a, 0x872371aa, 0xba43581a, 0x9932774d, 0xa4525efd, 0xe3f2242d, - 0xde920d9d, 0x6cb2d18d, 0x51d2f83d, 0x167282ed, 0x2b12ab5d, 0xa9423c8c, 0x9422153c, 0xd3826fec, 0xeee2465c, 0x5cc29a4c, 0x61a2b3fc, - 0x2602c92c, 0x1b62e09c, 0xf9d2e0cf, 0xc4b2c97f, 0x8312b3af, 0xbe729a1f, 0x0c52460f, 0x31326fbf, 0x7692156f, 0x4bf23cdf, 0xc9a2ab0e, - 0xf4c282be, 0xb362f86e, 0x8e02d1de, 0x3c220dce, 0x0142247e, 0x46e25eae, 0x7b82771e, 0xb1e6b092, 0x8c869922, 0xcb26e3f2, 0xf646ca42, - 0x44661652, 0x79063fe2, 0x3ea64532, 0x03c66c82, 0x8196fb53, 0xbcf6d2e3, 0xfb56a833, 0xc6368183, 0x74165d93, 0x49767423, 0x0ed60ef3, - 0x33b62743, 0xd1062710, 0xec660ea0, 0xabc67470, 0x96a65dc0, 0x248681d0, 0x19e6a860, 0x5e46d2b0, 0x6326fb00, 0xe1766cd1, 0xdc164561, - 0x9bb63fb1, 0xa6d61601, 0x14f6ca11, 0x2996e3a1, 0x6e369971, 0x5356b0c1, 0x70279f96, 0x4d47b626, 0x0ae7ccf6, 0x3787e546, 0x85a73956, - 0xb8c710e6, 0xff676a36, 0xc2074386, 0x4057d457, 0x7d37fde7, 0x3a978737, 0x07f7ae87, 0xb5d77297, 0x88b75b27, 0xcf1721f7, 0xf2770847, - 0x10c70814, 0x2da721a4, 0x6a075b74, 0x576772c4, 0xe547aed4, 0xd8278764, 0x9f87fdb4, 0xa2e7d404, 0x20b743d5, 0x1dd76a65, 0x5a7710b5, - 0x67173905, 0xd537e515, 0xe857cca5, 0xaff7b675, 0x92979fc5, 0xe915e8db, 0xd475c16b, 0x93d5bbbb, 0xaeb5920b, 0x1c954e1b, 0x21f567ab, - 0x66551d7b, 0x5b3534cb, 0xd965a31a, 0xe4058aaa, 0xa3a5f07a, 0x9ec5d9ca, 0x2ce505da, 0x11852c6a, 0x562556ba, 0x6b457f0a, 0x89f57f59, - 0xb49556e9, 0xf3352c39, 0xce550589, 0x7c75d999, 0x4115f029, 0x06b58af9, 0x3bd5a349, 0xb9853498, 0x84e51d28, 0xc34567f8, 0xfe254e48, - 0x4c059258, 0x7165bbe8, 0x36c5c138, 0x0ba5e888, 0x28d4c7df, 0x15b4ee6f, 0x521494bf, 0x6f74bd0f, 0xdd54611f, 0xe03448af, 0xa794327f, - 0x9af41bcf, 0x18a48c1e, 0x25c4a5ae, 0x6264df7e, 0x5f04f6ce, 0xed242ade, 0xd044036e, 0x97e479be, 0xaa84500e, 0x4834505d, 0x755479ed, - 0x32f4033d, 0x0f942a8d, 0xbdb4f69d, 0x80d4df2d, 0xc774a5fd, 0xfa148c4d, 0x78441b9c, 0x4524322c, 0x028448fc, 0x3fe4614c, 0x8dc4bd5c, - 0xb0a494ec, 0xf704ee3c, 0xca64c78c}, - {0x00000000, 0xcb5cd3a5, 0x4dc8a10b, 0x869472ae, 0x9b914216, 0x50cd91b3, 0xd659e31d, 0x1d0530b8, 0xec53826d, 0x270f51c8, 0xa19b2366, - 0x6ac7f0c3, 0x77c2c07b, 0xbc9e13de, 0x3a0a6170, 0xf156b2d5, 0x03d6029b, 0xc88ad13e, 0x4e1ea390, 0x85427035, 0x9847408d, 0x531b9328, - 0xd58fe186, 0x1ed33223, 0xef8580f6, 0x24d95353, 0xa24d21fd, 0x6911f258, 0x7414c2e0, 0xbf481145, 0x39dc63eb, 0xf280b04e, 0x07ac0536, - 0xccf0d693, 0x4a64a43d, 0x81387798, 0x9c3d4720, 0x57619485, 0xd1f5e62b, 0x1aa9358e, 0xebff875b, 0x20a354fe, 0xa6372650, 0x6d6bf5f5, - 0x706ec54d, 0xbb3216e8, 0x3da66446, 0xf6fab7e3, 0x047a07ad, 0xcf26d408, 0x49b2a6a6, 0x82ee7503, 0x9feb45bb, 0x54b7961e, 0xd223e4b0, - 0x197f3715, 0xe82985c0, 0x23755665, 0xa5e124cb, 0x6ebdf76e, 0x73b8c7d6, 0xb8e41473, 0x3e7066dd, 0xf52cb578, 0x0f580a6c, 0xc404d9c9, - 0x4290ab67, 0x89cc78c2, 0x94c9487a, 0x5f959bdf, 0xd901e971, 0x125d3ad4, 0xe30b8801, 0x28575ba4, 0xaec3290a, 0x659ffaaf, 0x789aca17, - 0xb3c619b2, 0x35526b1c, 0xfe0eb8b9, 0x0c8e08f7, 0xc7d2db52, 0x4146a9fc, 0x8a1a7a59, 0x971f4ae1, 0x5c439944, 0xdad7ebea, 0x118b384f, - 0xe0dd8a9a, 0x2b81593f, 0xad152b91, 0x6649f834, 0x7b4cc88c, 0xb0101b29, 0x36846987, 0xfdd8ba22, 0x08f40f5a, 0xc3a8dcff, 0x453cae51, - 0x8e607df4, 0x93654d4c, 0x58399ee9, 0xdeadec47, 0x15f13fe2, 0xe4a78d37, 0x2ffb5e92, 0xa96f2c3c, 0x6233ff99, 0x7f36cf21, 0xb46a1c84, - 0x32fe6e2a, 0xf9a2bd8f, 0x0b220dc1, 0xc07ede64, 0x46eaacca, 0x8db67f6f, 0x90b34fd7, 0x5bef9c72, 0xdd7beedc, 0x16273d79, 0xe7718fac, - 0x2c2d5c09, 0xaab92ea7, 0x61e5fd02, 0x7ce0cdba, 0xb7bc1e1f, 0x31286cb1, 0xfa74bf14, 0x1eb014d8, 0xd5ecc77d, 0x5378b5d3, 0x98246676, - 0x852156ce, 0x4e7d856b, 0xc8e9f7c5, 0x03b52460, 0xf2e396b5, 0x39bf4510, 0xbf2b37be, 0x7477e41b, 0x6972d4a3, 0xa22e0706, 0x24ba75a8, - 0xefe6a60d, 0x1d661643, 0xd63ac5e6, 0x50aeb748, 0x9bf264ed, 0x86f75455, 0x4dab87f0, 0xcb3ff55e, 0x006326fb, 0xf135942e, 0x3a69478b, - 0xbcfd3525, 0x77a1e680, 0x6aa4d638, 0xa1f8059d, 0x276c7733, 0xec30a496, 0x191c11ee, 0xd240c24b, 0x54d4b0e5, 0x9f886340, 0x828d53f8, - 0x49d1805d, 0xcf45f2f3, 0x04192156, 0xf54f9383, 0x3e134026, 0xb8873288, 0x73dbe12d, 0x6eded195, 0xa5820230, 0x2316709e, 0xe84aa33b, - 0x1aca1375, 0xd196c0d0, 0x5702b27e, 0x9c5e61db, 0x815b5163, 0x4a0782c6, 0xcc93f068, 0x07cf23cd, 0xf6999118, 0x3dc542bd, 0xbb513013, - 0x700de3b6, 0x6d08d30e, 0xa65400ab, 0x20c07205, 0xeb9ca1a0, 0x11e81eb4, 0xdab4cd11, 0x5c20bfbf, 0x977c6c1a, 0x8a795ca2, 0x41258f07, - 0xc7b1fda9, 0x0ced2e0c, 0xfdbb9cd9, 0x36e74f7c, 0xb0733dd2, 0x7b2fee77, 0x662adecf, 0xad760d6a, 0x2be27fc4, 0xe0beac61, 0x123e1c2f, - 0xd962cf8a, 0x5ff6bd24, 0x94aa6e81, 0x89af5e39, 0x42f38d9c, 0xc467ff32, 0x0f3b2c97, 0xfe6d9e42, 0x35314de7, 0xb3a53f49, 0x78f9ecec, - 0x65fcdc54, 0xaea00ff1, 0x28347d5f, 0xe368aefa, 0x16441b82, 0xdd18c827, 0x5b8cba89, 0x90d0692c, 0x8dd55994, 0x46898a31, 0xc01df89f, - 0x0b412b3a, 0xfa1799ef, 0x314b4a4a, 0xb7df38e4, 0x7c83eb41, 0x6186dbf9, 0xaada085c, 0x2c4e7af2, 0xe712a957, 0x15921919, 0xdececabc, - 0x585ab812, 0x93066bb7, 0x8e035b0f, 0x455f88aa, 0xc3cbfa04, 0x089729a1, 0xf9c19b74, 0x329d48d1, 0xb4093a7f, 0x7f55e9da, 0x6250d962, - 0xa90c0ac7, 0x2f987869, 0xe4c4abcc}, - {0x00000000, 0xa6770bb4, 0x979f1129, 0x31e81a9d, 0xf44f2413, 0x52382fa7, 0x63d0353a, 0xc5a73e8e, 0x33ef4e67, 0x959845d3, 0xa4705f4e, - 0x020754fa, 0xc7a06a74, 0x61d761c0, 0x503f7b5d, 0xf64870e9, 0x67de9cce, 0xc1a9977a, 0xf0418de7, 0x56368653, 0x9391b8dd, 0x35e6b369, - 0x040ea9f4, 0xa279a240, 0x5431d2a9, 0xf246d91d, 0xc3aec380, 0x65d9c834, 0xa07ef6ba, 0x0609fd0e, 0x37e1e793, 0x9196ec27, 0xcfbd399c, - 0x69ca3228, 0x582228b5, 0xfe552301, 0x3bf21d8f, 0x9d85163b, 0xac6d0ca6, 0x0a1a0712, 0xfc5277fb, 0x5a257c4f, 0x6bcd66d2, 0xcdba6d66, - 0x081d53e8, 0xae6a585c, 0x9f8242c1, 0x39f54975, 0xa863a552, 0x0e14aee6, 0x3ffcb47b, 0x998bbfcf, 0x5c2c8141, 0xfa5b8af5, 0xcbb39068, - 0x6dc49bdc, 0x9b8ceb35, 0x3dfbe081, 0x0c13fa1c, 0xaa64f1a8, 0x6fc3cf26, 0xc9b4c492, 0xf85cde0f, 0x5e2bd5bb, 0x440b7579, 0xe27c7ecd, - 0xd3946450, 0x75e36fe4, 0xb044516a, 0x16335ade, 0x27db4043, 0x81ac4bf7, 0x77e43b1e, 0xd19330aa, 0xe07b2a37, 0x460c2183, 0x83ab1f0d, - 0x25dc14b9, 0x14340e24, 0xb2430590, 0x23d5e9b7, 0x85a2e203, 0xb44af89e, 0x123df32a, 0xd79acda4, 0x71edc610, 0x4005dc8d, 0xe672d739, - 0x103aa7d0, 0xb64dac64, 0x87a5b6f9, 0x21d2bd4d, 0xe47583c3, 0x42028877, 0x73ea92ea, 0xd59d995e, 0x8bb64ce5, 0x2dc14751, 0x1c295dcc, - 0xba5e5678, 0x7ff968f6, 0xd98e6342, 0xe86679df, 0x4e11726b, 0xb8590282, 0x1e2e0936, 0x2fc613ab, 0x89b1181f, 0x4c162691, 0xea612d25, - 0xdb8937b8, 0x7dfe3c0c, 0xec68d02b, 0x4a1fdb9f, 0x7bf7c102, 0xdd80cab6, 0x1827f438, 0xbe50ff8c, 0x8fb8e511, 0x29cfeea5, 0xdf879e4c, - 0x79f095f8, 0x48188f65, 0xee6f84d1, 0x2bc8ba5f, 0x8dbfb1eb, 0xbc57ab76, 0x1a20a0c2, 0x8816eaf2, 0x2e61e146, 0x1f89fbdb, 0xb9fef06f, - 0x7c59cee1, 0xda2ec555, 0xebc6dfc8, 0x4db1d47c, 0xbbf9a495, 0x1d8eaf21, 0x2c66b5bc, 0x8a11be08, 0x4fb68086, 0xe9c18b32, 0xd82991af, - 0x7e5e9a1b, 0xefc8763c, 0x49bf7d88, 0x78576715, 0xde206ca1, 0x1b87522f, 0xbdf0599b, 0x8c184306, 0x2a6f48b2, 0xdc27385b, 0x7a5033ef, - 0x4bb82972, 0xedcf22c6, 0x28681c48, 0x8e1f17fc, 0xbff70d61, 0x198006d5, 0x47abd36e, 0xe1dcd8da, 0xd034c247, 0x7643c9f3, 0xb3e4f77d, - 0x1593fcc9, 0x247be654, 0x820cede0, 0x74449d09, 0xd23396bd, 0xe3db8c20, 0x45ac8794, 0x800bb91a, 0x267cb2ae, 0x1794a833, 0xb1e3a387, - 0x20754fa0, 0x86024414, 0xb7ea5e89, 0x119d553d, 0xd43a6bb3, 0x724d6007, 0x43a57a9a, 0xe5d2712e, 0x139a01c7, 0xb5ed0a73, 0x840510ee, - 0x22721b5a, 0xe7d525d4, 0x41a22e60, 0x704a34fd, 0xd63d3f49, 0xcc1d9f8b, 0x6a6a943f, 0x5b828ea2, 0xfdf58516, 0x3852bb98, 0x9e25b02c, - 0xafcdaab1, 0x09baa105, 0xfff2d1ec, 0x5985da58, 0x686dc0c5, 0xce1acb71, 0x0bbdf5ff, 0xadcafe4b, 0x9c22e4d6, 0x3a55ef62, 0xabc30345, - 0x0db408f1, 0x3c5c126c, 0x9a2b19d8, 0x5f8c2756, 0xf9fb2ce2, 0xc813367f, 0x6e643dcb, 0x982c4d22, 0x3e5b4696, 0x0fb35c0b, 0xa9c457bf, - 0x6c636931, 0xca146285, 0xfbfc7818, 0x5d8b73ac, 0x03a0a617, 0xa5d7ada3, 0x943fb73e, 0x3248bc8a, 0xf7ef8204, 0x519889b0, 0x6070932d, - 0xc6079899, 0x304fe870, 0x9638e3c4, 0xa7d0f959, 0x01a7f2ed, 0xc400cc63, 0x6277c7d7, 0x539fdd4a, 0xf5e8d6fe, 0x647e3ad9, 0xc209316d, - 0xf3e12bf0, 0x55962044, 0x90311eca, 0x3646157e, 0x07ae0fe3, 0xa1d90457, 0x579174be, 0xf1e67f0a, 0xc00e6597, 0x66796e23, 0xa3de50ad, - 0x05a95b19, 0x34414184, 0x92364a30}, - {0x00000000, 0xccaa009e, 0x4225077d, 0x8e8f07e3, 0x844a0efa, 0x48e00e64, 0xc66f0987, 0x0ac50919, 0xd3e51bb5, 0x1f4f1b2b, 0x91c01cc8, - 0x5d6a1c56, 0x57af154f, 0x9b0515d1, 0x158a1232, 0xd92012ac, 0x7cbb312b, 0xb01131b5, 0x3e9e3656, 0xf23436c8, 0xf8f13fd1, 0x345b3f4f, - 0xbad438ac, 0x767e3832, 0xaf5e2a9e, 0x63f42a00, 0xed7b2de3, 0x21d12d7d, 0x2b142464, 0xe7be24fa, 0x69312319, 0xa59b2387, 0xf9766256, - 0x35dc62c8, 0xbb53652b, 0x77f965b5, 0x7d3c6cac, 0xb1966c32, 0x3f196bd1, 0xf3b36b4f, 0x2a9379e3, 0xe639797d, 0x68b67e9e, 0xa41c7e00, - 0xaed97719, 0x62737787, 0xecfc7064, 0x205670fa, 0x85cd537d, 0x496753e3, 0xc7e85400, 0x0b42549e, 0x01875d87, 0xcd2d5d19, 0x43a25afa, - 0x8f085a64, 0x562848c8, 0x9a824856, 0x140d4fb5, 0xd8a74f2b, 0xd2624632, 0x1ec846ac, 0x9047414f, 0x5ced41d1, 0x299dc2ed, 0xe537c273, - 0x6bb8c590, 0xa712c50e, 0xadd7cc17, 0x617dcc89, 0xeff2cb6a, 0x2358cbf4, 0xfa78d958, 0x36d2d9c6, 0xb85dde25, 0x74f7debb, 0x7e32d7a2, - 0xb298d73c, 0x3c17d0df, 0xf0bdd041, 0x5526f3c6, 0x998cf358, 0x1703f4bb, 0xdba9f425, 0xd16cfd3c, 0x1dc6fda2, 0x9349fa41, 0x5fe3fadf, - 0x86c3e873, 0x4a69e8ed, 0xc4e6ef0e, 0x084cef90, 0x0289e689, 0xce23e617, 0x40ace1f4, 0x8c06e16a, 0xd0eba0bb, 0x1c41a025, 0x92cea7c6, - 0x5e64a758, 0x54a1ae41, 0x980baedf, 0x1684a93c, 0xda2ea9a2, 0x030ebb0e, 0xcfa4bb90, 0x412bbc73, 0x8d81bced, 0x8744b5f4, 0x4beeb56a, - 0xc561b289, 0x09cbb217, 0xac509190, 0x60fa910e, 0xee7596ed, 0x22df9673, 0x281a9f6a, 0xe4b09ff4, 0x6a3f9817, 0xa6959889, 0x7fb58a25, - 0xb31f8abb, 0x3d908d58, 0xf13a8dc6, 0xfbff84df, 0x37558441, 0xb9da83a2, 0x7570833c, 0x533b85da, 0x9f918544, 0x111e82a7, 0xddb48239, - 0xd7718b20, 0x1bdb8bbe, 0x95548c5d, 0x59fe8cc3, 0x80de9e6f, 0x4c749ef1, 0xc2fb9912, 0x0e51998c, 0x04949095, 0xc83e900b, 0x46b197e8, - 0x8a1b9776, 0x2f80b4f1, 0xe32ab46f, 0x6da5b38c, 0xa10fb312, 0xabcaba0b, 0x6760ba95, 0xe9efbd76, 0x2545bde8, 0xfc65af44, 0x30cfafda, - 0xbe40a839, 0x72eaa8a7, 0x782fa1be, 0xb485a120, 0x3a0aa6c3, 0xf6a0a65d, 0xaa4de78c, 0x66e7e712, 0xe868e0f1, 0x24c2e06f, 0x2e07e976, - 0xe2ade9e8, 0x6c22ee0b, 0xa088ee95, 0x79a8fc39, 0xb502fca7, 0x3b8dfb44, 0xf727fbda, 0xfde2f2c3, 0x3148f25d, 0xbfc7f5be, 0x736df520, - 0xd6f6d6a7, 0x1a5cd639, 0x94d3d1da, 0x5879d144, 0x52bcd85d, 0x9e16d8c3, 0x1099df20, 0xdc33dfbe, 0x0513cd12, 0xc9b9cd8c, 0x4736ca6f, - 0x8b9ccaf1, 0x8159c3e8, 0x4df3c376, 0xc37cc495, 0x0fd6c40b, 0x7aa64737, 0xb60c47a9, 0x3883404a, 0xf42940d4, 0xfeec49cd, 0x32464953, - 0xbcc94eb0, 0x70634e2e, 0xa9435c82, 0x65e95c1c, 0xeb665bff, 0x27cc5b61, 0x2d095278, 0xe1a352e6, 0x6f2c5505, 0xa386559b, 0x061d761c, - 0xcab77682, 0x44387161, 0x889271ff, 0x825778e6, 0x4efd7878, 0xc0727f9b, 0x0cd87f05, 0xd5f86da9, 0x19526d37, 0x97dd6ad4, 0x5b776a4a, - 0x51b26353, 0x9d1863cd, 0x1397642e, 0xdf3d64b0, 0x83d02561, 0x4f7a25ff, 0xc1f5221c, 0x0d5f2282, 0x079a2b9b, 0xcb302b05, 0x45bf2ce6, - 0x89152c78, 0x50353ed4, 0x9c9f3e4a, 0x121039a9, 0xdeba3937, 0xd47f302e, 0x18d530b0, 0x965a3753, 0x5af037cd, 0xff6b144a, 0x33c114d4, - 0xbd4e1337, 0x71e413a9, 0x7b211ab0, 0xb78b1a2e, 0x39041dcd, 0xf5ae1d53, 0x2c8e0fff, 0xe0240f61, 0x6eab0882, 0xa201081c, 0xa8c40105, - 0x646e019b, 0xeae10678, 0x264b06e6}}; - -#define BYTESWAP_ORDER32(x) (((x) >> 24) + (((x) >> 8) & 0xff00) + (((x) << 8) & 0xff0000) + ((x) << 24)) -#define UE_PTRDIFF_TO_UINT32(argument) static_cast<uint32_t>(argument) - -template<typename T> -constexpr T -Align(T Val, uint64_t Alignment) -{ - return (T)(((uint64_t)Val + Alignment - 1) & ~(Alignment - 1)); -} - -} // namespace CRC32 - -namespace zen { - -uint32_t -StrCrc_Deprecated(const char* Data) -{ - using namespace CRC32; - - uint32_t CRC = 0xFFFFFFFF; - while (*Data) - { - char16_t C = *Data++; - int32_t CL = (C & 255); - CRC = (CRC << 8) ^ CRCTable_DEPRECATED[(CRC >> 24) ^ CL]; - int32_t CH = (C >> 8) & 255; - CRC = (CRC << 8) ^ CRCTable_DEPRECATED[(CRC >> 24) ^ CH]; - } - return ~CRC; -} - -uint32_t -MemCrc32(const void* InData, size_t Length, uint32_t CRC /*=0 */) -{ - using namespace CRC32; - - // Based on the Slicing-by-8 implementation found here: - // http://slicing-by-8.sourceforge.net/ - - CRC = ~CRC; - - const uint8_t* __restrict Data = (uint8_t*)InData; - - // First we need to align to 32-bits - uint32_t InitBytes = UE_PTRDIFF_TO_UINT32(Align(Data, 4) - Data); - - if (Length > InitBytes) - { - Length -= InitBytes; - - for (; InitBytes; --InitBytes) - { - CRC = (CRC >> 8) ^ CRCTablesSB8[0][(CRC & 0xFF) ^ *Data++]; - } - - auto Data4 = (const uint32_t*)Data; - for (size_t Repeat = Length / 8; Repeat; --Repeat) - { - uint32_t V1 = *Data4++ ^ CRC; - uint32_t V2 = *Data4++; - CRC = CRCTablesSB8[7][V1 & 0xFF] ^ CRCTablesSB8[6][(V1 >> 8) & 0xFF] ^ CRCTablesSB8[5][(V1 >> 16) & 0xFF] ^ - CRCTablesSB8[4][V1 >> 24] ^ CRCTablesSB8[3][V2 & 0xFF] ^ CRCTablesSB8[2][(V2 >> 8) & 0xFF] ^ - CRCTablesSB8[1][(V2 >> 16) & 0xFF] ^ CRCTablesSB8[0][V2 >> 24]; - } - Data = (const uint8_t*)Data4; - - Length %= 8; - } - - for (; Length; --Length) - { - CRC = (CRC >> 8) ^ CRCTablesSB8[0][(CRC & 0xFF) ^ *Data++]; - } - - return ~CRC; -} - -uint32_t -MemCrc32_Deprecated(const void* InData, size_t Length, uint32_t CRC) -{ - using namespace CRC32; - - // Based on the Slicing-by-8 implementation found here: - // http://slicing-by-8.sourceforge.net/ - - CRC = ~BYTESWAP_ORDER32(CRC); - - const uint8_t* __restrict Data = (uint8_t*)InData; - - // First we need to align to 32-bits - uint32_t InitBytes = UE_PTRDIFF_TO_UINT32(Align(Data, 4) - Data); - - if (Length > InitBytes) - { - Length -= InitBytes; - - for (; InitBytes; --InitBytes) - { - CRC = (CRC >> 8) ^ CRCTablesSB8_DEPRECATED[0][(CRC & 0xFF) ^ *Data++]; - } - - auto Data4 = (const uint32_t*)Data; - for (size_t Repeat = Length / 8; Repeat; --Repeat) - { - uint32_t V1 = *Data4++ ^ CRC; - uint32_t V2 = *Data4++; - CRC = CRCTablesSB8_DEPRECATED[7][V1 & 0xFF] ^ CRCTablesSB8_DEPRECATED[6][(V1 >> 8) & 0xFF] ^ - CRCTablesSB8_DEPRECATED[5][(V1 >> 16) & 0xFF] ^ CRCTablesSB8_DEPRECATED[4][V1 >> 24] ^ - CRCTablesSB8_DEPRECATED[3][V2 & 0xFF] ^ CRCTablesSB8_DEPRECATED[2][(V2 >> 8) & 0xFF] ^ - CRCTablesSB8_DEPRECATED[1][(V2 >> 16) & 0xFF] ^ CRCTablesSB8_DEPRECATED[0][V2 >> 24]; - } - Data = (const uint8_t*)Data4; - - Length %= 8; - } - - for (; Length; --Length) - { - CRC = (CRC >> 8) ^ CRCTablesSB8_DEPRECATED[0][(CRC & 0xFF) ^ *Data++]; - } - - return BYTESWAP_ORDER32(~CRC); -} - -} // namespace zen diff --git a/zencore/crypto.cpp b/zencore/crypto.cpp deleted file mode 100644 index 448fd36fa..000000000 --- a/zencore/crypto.cpp +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/crypto.h> -#include <zencore/intmath.h> -#include <zencore/testing.h> - -#include <string> -#include <string_view> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <fmt/format.h> -#include <openssl/conf.h> -#include <openssl/err.h> -#include <openssl/evp.h> -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_PLATFORM_WINDOWS -# pragma comment(lib, "crypt32.lib") -# pragma comment(lib, "ws2_32.lib") -#endif - -namespace zen { - -using namespace std::literals; - -namespace crypto { - - class EvpContext - { - public: - EvpContext() : m_Ctx(EVP_CIPHER_CTX_new()) {} - ~EvpContext() { EVP_CIPHER_CTX_free(m_Ctx); } - - operator EVP_CIPHER_CTX*() { return m_Ctx; } - - private: - EVP_CIPHER_CTX* m_Ctx; - }; - - enum class TransformMode : uint32_t - { - Decrypt, - Encrypt - }; - - MemoryView Transform(const EVP_CIPHER* Cipher, - TransformMode Mode, - MemoryView Key, - MemoryView IV, - MemoryView In, - MutableMemoryView Out, - std::optional<std::string>& Reason) - { - ZEN_ASSERT(Cipher != nullptr); - - EvpContext Ctx; - - int Err = EVP_CipherInit_ex(Ctx, - Cipher, - nullptr, - reinterpret_cast<const unsigned char*>(Key.GetData()), - reinterpret_cast<const unsigned char*>(IV.GetData()), - static_cast<int>(Mode)); - - if (Err != 1) - { - Reason = fmt::format("failed to initialize cipher, error code '{}'", Err); - - return MemoryView(); - } - - int EncryptedBytes = 0; - int TotalEncryptedBytes = 0; - - Err = EVP_CipherUpdate(Ctx, - reinterpret_cast<unsigned char*>(Out.GetData()), - &EncryptedBytes, - reinterpret_cast<const unsigned char*>(In.GetData()), - static_cast<int>(In.GetSize())); - - if (Err != 1) - { - Reason = fmt::format("update crypto transform failed, error code '{}'", Err); - - return MemoryView(); - } - - TotalEncryptedBytes = EncryptedBytes; - MutableMemoryView Remaining = Out.RightChop(EncryptedBytes); - - EncryptedBytes = static_cast<int>(Remaining.GetSize()); - - Err = EVP_CipherFinal(Ctx, reinterpret_cast<unsigned char*>(Remaining.GetData()), &EncryptedBytes); - - if (Err != 1) - { - Reason = fmt::format("finalize crypto transform failed, error code '{}'", Err); - - return MemoryView(); - } - - TotalEncryptedBytes += EncryptedBytes; - - return Out.Left(TotalEncryptedBytes); - } - - bool ValidateKeyAndIV(const AesKey256Bit& Key, const AesIV128Bit& IV, std::optional<std::string>& Reason) - { - if (Key.IsValid() == false) - { - Reason = "invalid key"sv; - - return false; - } - - if (IV.IsValid() == false) - { - Reason = "invalid initialization vector"sv; - - return false; - } - - return true; - } - -} // namespace crypto - -MemoryView -Aes::Encrypt(const AesKey256Bit& Key, const AesIV128Bit& IV, MemoryView In, MutableMemoryView Out, std::optional<std::string>& Reason) -{ - if (crypto::ValidateKeyAndIV(Key, IV, Reason) == false) - { - return MemoryView(); - } - - return crypto::Transform(EVP_aes_256_cbc(), crypto::TransformMode::Encrypt, Key.GetView(), IV.GetView(), In, Out, Reason); -} - -MemoryView -Aes::Decrypt(const AesKey256Bit& Key, const AesIV128Bit& IV, MemoryView In, MutableMemoryView Out, std::optional<std::string>& Reason) -{ - if (crypto::ValidateKeyAndIV(Key, IV, Reason) == false) - { - return MemoryView(); - } - - return crypto::Transform(EVP_aes_256_cbc(), crypto::TransformMode::Decrypt, Key.GetView(), IV.GetView(), In, Out, Reason); -} - -#if ZEN_WITH_TESTS - -void -crypto_forcelink() -{ -} - -TEST_CASE("crypto.bits") -{ - using CryptoBits256Bit = CryptoBits<256>; - - CryptoBits256Bit Bits; - - CHECK(Bits.IsNull()); - CHECK(Bits.IsValid() == false); - - CHECK(Bits.GetBitCount() == 256); - CHECK(Bits.GetSize() == 32); - - Bits = CryptoBits256Bit::FromString("Addff"sv); - CHECK(Bits.IsValid() == false); - - Bits = CryptoBits256Bit::FromString("abcdefghijklmnopqrstuvxyz0123456"sv); - CHECK(Bits.IsValid()); - - auto SmallerBits = CryptoBits<128>::FromString("abcdefghijklmnopqrstuvxyz0123456"sv); - CHECK(SmallerBits.IsValid() == false); -} - -TEST_CASE("crypto.aes") -{ - SUBCASE("basic") - { - const uint8_t InitVector[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; - const AesKey256Bit Key = AesKey256Bit::FromString("abcdefghijklmnopqrstuvxyz0123456"sv); - const AesIV128Bit IV = AesIV128Bit::FromMemoryView(MakeMemoryView(InitVector)); - - std::string_view PlainText = "The quick brown fox jumps over the lazy dog"sv; - - std::vector<uint8_t> EncryptionBuffer; - std::vector<uint8_t> DecryptionBuffer; - std::optional<std::string> Reason; - - EncryptionBuffer.resize(PlainText.size() + Aes::BlockSize); - DecryptionBuffer.resize(PlainText.size() + Aes::BlockSize); - - MemoryView EncryptedView = Aes::Encrypt(Key, IV, MakeMemoryView(PlainText), MakeMutableMemoryView(EncryptionBuffer), Reason); - MemoryView DecryptedView = Aes::Decrypt(Key, IV, EncryptedView, MakeMutableMemoryView(DecryptionBuffer), Reason); - - std::string_view EncryptedDecryptedText = - std::string_view(reinterpret_cast<const char*>(DecryptedView.GetData()), DecryptedView.GetSize()); - - CHECK(EncryptedDecryptedText == PlainText); - } -} - -#endif - -} // namespace zen diff --git a/zencore/except.cpp b/zencore/except.cpp deleted file mode 100644 index 2749d1984..000000000 --- a/zencore/except.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <fmt/format.h> -#include <zencore/except.h> - -namespace zen { - -#if ZEN_PLATFORM_WINDOWS - -class WindowsException : public std::exception -{ -public: - WindowsException(std::string_view Message) - { - m_hResult = HRESULT_FROM_WIN32(GetLastError()); - m_Message = Message; - } - - WindowsException(HRESULT hRes, std::string_view Message) - { - m_hResult = hRes; - m_Message = Message; - } - - WindowsException(HRESULT hRes, const char* Message, const char* Detail) - { - m_hResult = hRes; - - ExtendableStringBuilder<128> msg; - msg.Append(Message); - msg.Append(" (detail: '"); - msg.Append(Detail); - msg.Append("')"); - - m_Message = msg.c_str(); - } - - virtual const char* what() const override { return m_Message.c_str(); } - -private: - std::string m_Message; - HRESULT m_hResult; -}; - -void -ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string_view Message) -{ - if (HRESULT_FACILITY(hRes) == FACILITY_WIN32) - { - throw std::system_error(std::error_code(hRes & 0xffff, std::system_category()), std::string(Message)); - } - else - { - throw WindowsException(hRes, Message); - } -} - -#endif // ZEN_PLATFORM_WINDOWS - -void -ThrowSystemError(uint32_t ErrorCode, std::string_view Message) -{ - throw std::system_error(std::error_code(ErrorCode, std::system_category()), std::string(Message)); -} - -std::string -GetLastErrorAsString() -{ - return GetSystemErrorAsString(zen::GetLastError()); -} - -std::string -GetSystemErrorAsString(uint32_t ErrorCode) -{ - return std::error_code(ErrorCode, std::system_category()).message(); -} - -#if defined(__cpp_lib_source_location) -void -ThrowLastErrorImpl(std::string_view Message, const std::source_location& Location) -{ - throw std::system_error(std::error_code(zen::GetLastError(), std::system_category()), - fmt::format("{}({}): {}", Location.file_name(), Location.line(), Message)); -} -#else -void -ThrowLastError(std::string_view Message) -{ - throw std::system_error(std::error_code(zen::GetLastError(), std::system_category()), std::string(Message)); -} -#endif - -} // namespace zen diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp deleted file mode 100644 index a17773024..000000000 --- a/zencore/filesystem.cpp +++ /dev/null @@ -1,1304 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/filesystem.h> - -#include <zencore/except.h> -#include <zencore/fmtutils.h> -#include <zencore/iobuffer.h> -#include <zencore/logging.h> -#include <zencore/stream.h> -#include <zencore/string.h> -#include <zencore/testing.h> - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#endif - -#if ZEN_PLATFORM_WINDOWS -# include <atlbase.h> -# include <atlfile.h> -# include <winioctl.h> -# include <winnt.h> -#endif - -#if ZEN_PLATFORM_LINUX -# include <dirent.h> -# include <fcntl.h> -# include <sys/resource.h> -# include <sys/stat.h> -# include <unistd.h> -#endif - -#if ZEN_PLATFORM_MAC -# include <dirent.h> -# include <fcntl.h> -# include <libproc.h> -# include <sys/resource.h> -# include <sys/stat.h> -# include <sys/syslimits.h> -# include <unistd.h> -#endif - -#include <filesystem> -#include <gsl/gsl-lite.hpp> - -namespace zen { - -using namespace std::literals; - -#if ZEN_PLATFORM_WINDOWS - -static bool -DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) -{ - CHandle hDir(CreateFileW(Path, - GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, - nullptr)); - - if (hDir != INVALID_HANDLE_VALUE) - { - REPARSE_GUID_DATA_BUFFER Rgdb = {}; - Rgdb.ReparseTag = dwReparseTag; - - DWORD dwBytes; - const BOOL bOK = - DeviceIoControl(hDir, FSCTL_DELETE_REPARSE_POINT, &Rgdb, REPARSE_GUID_DATA_BUFFER_HEADER_SIZE, nullptr, 0, &dwBytes, nullptr); - - return bOK == TRUE; - } - - return false; -} - -bool -CreateDirectories(const wchar_t* Dir) -{ - // This may be suboptimal, in that it appears to try and create directories - // from the root on up instead of from some directory which is known to - // be present - // - // We should implement a smarter version at some point since this can be - // pretty expensive in aggregate - - return std::filesystem::create_directories(Dir); -} - -// Erase all files and directories in a given directory, leaving an empty directory -// behind - -static bool -WipeDirectory(const wchar_t* DirPath) -{ - ExtendableWideStringBuilder<128> Pattern; - Pattern.Append(DirPath); - Pattern.Append(L"\\*"); - - WIN32_FIND_DATAW FindData; - HANDLE hFind = FindFirstFileW(Pattern.c_str(), &FindData); - - if (hFind != nullptr) - { - do - { - bool IsRegular = true; - - if (FindData.cFileName[0] == L'.') - { - if (FindData.cFileName[1] == L'.') - { - if (FindData.cFileName[2] == L'\0') - { - IsRegular = false; - } - } - else if (FindData.cFileName[1] == L'\0') - { - IsRegular = false; - } - } - - if (IsRegular) - { - ExtendableWideStringBuilder<128> Path; - Path.Append(DirPath); - Path.Append(L'\\'); - Path.Append(FindData.cFileName); - - // if (fd.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN) - // deleteReparsePoint(path.c_str(), fd.dwReserved0); - - if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - { - if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN) - { - DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); - } - - if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) - { - DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); - } - - bool Success = DeleteDirectories(Path.c_str()); - - if (!Success) - { - if (FindData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) - { - DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); - } - } - } - else - { - DeleteFileW(Path.c_str()); - } - } - } while (FindNextFileW(hFind, &FindData) == TRUE); - - FindClose(hFind); - } - - return true; -} - -bool -DeleteDirectories(const wchar_t* DirPath) -{ - return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE; -} - -bool -CleanDirectory(const wchar_t* DirPath) -{ - if (std::filesystem::exists(DirPath)) - { - return WipeDirectory(DirPath); - } - - return CreateDirectories(DirPath); -} - -#endif // ZEN_PLATFORM_WINDOWS - -bool -CreateDirectories(const std::filesystem::path& Dir) -{ - while (!std::filesystem::is_directory(Dir)) - { - if (Dir.has_parent_path()) - { - CreateDirectories(Dir.parent_path()); - } - std::error_code ErrorCode; - std::filesystem::create_directory(Dir, ErrorCode); - if (ErrorCode) - { - throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string())); - } - return true; - } - return false; -} - -bool -DeleteDirectories(const std::filesystem::path& Dir) -{ -#if ZEN_PLATFORM_WINDOWS - return DeleteDirectories(Dir.c_str()); -#else - std::error_code ErrorCode; - return std::filesystem::remove_all(Dir, ErrorCode); -#endif -} - -bool -CleanDirectory(const std::filesystem::path& Dir) -{ -#if ZEN_PLATFORM_WINDOWS - return CleanDirectory(Dir.c_str()); -#else - if (std::filesystem::exists(Dir)) - { - bool Success = true; - - std::error_code ErrorCode; - for (const auto& Item : std::filesystem::directory_iterator(Dir)) - { - Success &= std::filesystem::remove_all(Item, ErrorCode); - } - - return Success; - } - - return CreateDirectories(Dir); -#endif -} - -////////////////////////////////////////////////////////////////////////// - -bool -SupportsBlockRefCounting(std::filesystem::path Path) -{ -#if ZEN_PLATFORM_WINDOWS - ATL::CHandle Handle(CreateFileW(Path.c_str(), - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, - nullptr)); - - if (Handle == INVALID_HANDLE_VALUE) - { - Handle.Detach(); - return false; - } - - ULONG FileSystemFlags = 0; - if (!GetVolumeInformationByHandleW(Handle, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0)) - { - return false; - } - - if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING)) - { - return false; - } - - return true; -#else - ZEN_UNUSED(Path); - return false; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) -{ -#if ZEN_PLATFORM_WINDOWS - ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); - if (FromFile == INVALID_HANDLE_VALUE) - { - FromFile.Detach(); - return false; - } - - ULONG FileSystemFlags; - if (!GetVolumeInformationByHandleW(FromFile, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0)) - { - return false; - } - if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING)) - { - SetLastError(ERROR_NOT_CAPABLE); - return false; - } - - FILE_END_OF_FILE_INFO FileSize; - if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile)) - { - return false; - } - - FILE_BASIC_INFO BasicInfo; - if (!GetFileInformationByHandleEx(FromFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo)) - { - return false; - } - - DWORD dwBytesReturned = 0; - FSCTL_GET_INTEGRITY_INFORMATION_BUFFER GetIntegrityInfoBuffer; - if (!DeviceIoControl(FromFile, - FSCTL_GET_INTEGRITY_INFORMATION, - nullptr, - 0, - &GetIntegrityInfoBuffer, - sizeof GetIntegrityInfoBuffer, - &dwBytesReturned, - nullptr)) - { - return false; - } - - SetFileAttributesW(ToPath.c_str(), FILE_ATTRIBUTE_NORMAL); - - ATL::CHandle TargetFile(CreateFileW(ToPath.c_str(), - GENERIC_READ | GENERIC_WRITE | DELETE, - /* no sharing */ FILE_SHARE_READ, - nullptr, - OPEN_ALWAYS, - 0, - /* hTemplateFile */ FromFile)); - - if (TargetFile == INVALID_HANDLE_VALUE) - { - TargetFile.Detach(); - return false; - } - - // Delete target file when handle is closed (we only reset this if the copy succeeds) - FILE_DISPOSITION_INFO FileDisposition = {TRUE}; - if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)) - { - TargetFile.Close(); - DeleteFileW(ToPath.c_str()); - return false; - } - - // Make file sparse so we don't end up allocating space when we change the file size - if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &dwBytesReturned, nullptr)) - { - return false; - } - - // Copy integrity checking information - FSCTL_SET_INTEGRITY_INFORMATION_BUFFER IntegritySet = {GetIntegrityInfoBuffer.ChecksumAlgorithm, - GetIntegrityInfoBuffer.Reserved, - GetIntegrityInfoBuffer.Flags}; - if (!DeviceIoControl(TargetFile, FSCTL_SET_INTEGRITY_INFORMATION, &IntegritySet, sizeof IntegritySet, nullptr, 0, nullptr, nullptr)) - { - return false; - } - - // Resize file - note that the file is sparse at this point so no additional data will be written - if (!SetFileInformationByHandle(TargetFile, FileEndOfFileInfo, &FileSize, sizeof FileSize)) - { - return false; - } - - constexpr auto RoundToClusterSize = [](LONG64 FileSize, ULONG ClusterSize) -> LONG64 { - return (FileSize + ClusterSize - 1) / ClusterSize * ClusterSize; - }; - static_assert(RoundToClusterSize(5678, 4 * 1024) == 8 * 1024); - - // Loop for cloning file contents. This is necessary as the API has a 32-bit size - // limit for some reason - - const LONG64 SplitThreshold = (1LL << 32) - GetIntegrityInfoBuffer.ClusterSizeInBytes; - - DUPLICATE_EXTENTS_DATA DuplicateExtentsData{.FileHandle = FromFile}; - - for (LONG64 CurrentByteOffset = 0, - RemainingBytes = RoundToClusterSize(FileSize.EndOfFile.QuadPart, GetIntegrityInfoBuffer.ClusterSizeInBytes); - RemainingBytes > 0; - CurrentByteOffset += SplitThreshold, RemainingBytes -= SplitThreshold) - { - DuplicateExtentsData.SourceFileOffset.QuadPart = CurrentByteOffset; - DuplicateExtentsData.TargetFileOffset.QuadPart = CurrentByteOffset; - DuplicateExtentsData.ByteCount.QuadPart = std::min(SplitThreshold, RemainingBytes); - - if (!DeviceIoControl(TargetFile, - FSCTL_DUPLICATE_EXTENTS_TO_FILE, - &DuplicateExtentsData, - sizeof DuplicateExtentsData, - nullptr, - 0, - &dwBytesReturned, - nullptr)) - { - return false; - } - } - - // Make the file not sparse again now that we have populated the contents - if (!(BasicInfo.FileAttributes & FILE_ATTRIBUTE_SPARSE_FILE)) - { - FILE_SET_SPARSE_BUFFER SetSparse = {FALSE}; - - if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, &SetSparse, sizeof SetSparse, nullptr, 0, &dwBytesReturned, nullptr)) - { - return false; - } - } - - // Update timestamps (but don't lie about the creation time) - BasicInfo.CreationTime.QuadPart = 0; - if (!SetFileInformationByHandle(TargetFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo)) - { - return false; - } - - if (!FlushFileBuffers(TargetFile)) - { - return false; - } - - // Finally now everything is done - make sure the file is not deleted on close! - - FileDisposition = {FALSE}; - - const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)); - - return AllOk; -#elif ZEN_PLATFORM_LINUX -# if 0 - struct ScopedFd - { - ~ScopedFd() { close(Fd); } - int Fd; - }; - - // The 'from' file - int FromFd = open(FromPath.c_str(), O_RDONLY|O_CLOEXEC); - if (FromFd < 0) - { - return false; - } - ScopedFd $From = { FromFd }; - - // The 'to' file - int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC, 0666); - if (ToFd < 0) - { - return false; - } - fchmod(ToFd, 0666); - ScopedFd $To = { FromFd }; - - ioctl(ToFd, FICLONE, FromFd); - - return false; -# endif // 0 - ZEN_UNUSED(FromPath, ToPath); - ZEN_ERROR("CloneFile() is not implemented on this platform"); - return false; -#elif ZEN_PLATFORM_MAC - /* clonefile() syscall if APFS */ - ZEN_UNUSED(FromPath, ToPath); - ZEN_ERROR("CloneFile() is not implemented on this platform"); - return false; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) -{ - bool Success = false; - - if (Options.EnableClone) - { - Success = CloneFile(FromPath.native(), ToPath.native()); - - if (Success) - { - return true; - } - } - - if (Options.MustClone) - { - return false; - } - -#if ZEN_PLATFORM_WINDOWS - BOOL CancelFlag = FALSE; - Success = !!::CopyFileExW(FromPath.c_str(), - ToPath.c_str(), - /* lpProgressRoutine */ nullptr, - /* lpData */ nullptr, - &CancelFlag, - /* dwCopyFlags */ 0); -#else - struct ScopedFd - { - ~ScopedFd() { close(Fd); } - int Fd; - }; - - // From file - int FromFd = open(FromPath.c_str(), O_RDONLY | O_CLOEXEC); - if (FromFd < 0) - { - ThrowLastError(fmt::format("failed to open file {}", FromPath)); - } - ScopedFd $From = {FromFd}; - - // To file - int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666); - if (ToFd < 0) - { - ThrowLastError(fmt::format("failed to create file {}", ToPath)); - } - fchmod(ToFd, 0666); - ScopedFd $To = {ToFd}; - - // Copy impl - static const size_t BufferSize = 64 << 10; - void* Buffer = malloc(BufferSize); - while (true) - { - int BytesRead = read(FromFd, Buffer, BufferSize); - if (BytesRead <= 0) - { - Success = (BytesRead == 0); - break; - } - - if (write(ToFd, Buffer, BytesRead) != BufferSize) - { - Success = false; - break; - } - } - free(Buffer); -#endif // ZEN_PLATFORM_WINDOWS - - if (!Success) - { - ThrowLastError("file copy failed"sv); - } - - return true; -} - -void -WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount) -{ -#if ZEN_PLATFORM_WINDOWS - CAtlFile Outfile; - HRESULT hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); - if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND)) - { - CreateDirectories(Path.parent_path()); - - hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); - } - - if (FAILED(hRes)) - { - ThrowSystemException(hRes, fmt::format("File open failed for '{}'", Path).c_str()); - } - -#else - int OpenFlags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC; - int Fd = open(Path.c_str(), OpenFlags, 0666); - if (Fd < 0) - { - zen::CreateDirectories(Path.parent_path()); - Fd = open(Path.c_str(), OpenFlags, 0666); - } - - if (Fd < 0) - { - ThrowLastError(fmt::format("File open failed for '{}'", Path)); - } - - fchmod(Fd, 0666); -#endif - - // TODO: this should be block-enlightened - - for (size_t i = 0; i < BufferCount; ++i) - { - uint64_t WriteSize = Data[i]->Size(); - const void* DataPtr = Data[i]->Data(); - - while (WriteSize) - { - const uint64_t ChunkSize = Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024); - -#if ZEN_PLATFORM_WINDOWS - hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize)); - if (FAILED(hRes)) - { - ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); - } -#else - if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize)) - { - ThrowLastError(fmt::format("File write failed for '{}'", Path)); - } -#endif // ZEN_PLATFORM_WINDOWS - - WriteSize -= ChunkSize; - DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize; - } - } - -#if !ZEN_PLATFORM_WINDOWS - close(Fd); -#endif -} - -void -WriteFile(std::filesystem::path Path, IoBuffer Data) -{ - const IoBuffer* const DataPtr = &Data; - - WriteFile(Path, &DataPtr, 1); -} - -IoBuffer -FileContents::Flatten() -{ - if (Data.size() == 1) - { - return Data[0]; - } - else if (Data.empty()) - { - return {}; - } - else - { - ZEN_NOT_IMPLEMENTED(); - } -} - -FileContents -ReadStdIn() -{ - BinaryWriter Writer; - - do - { - uint8_t ReadBuffer[1024]; - - size_t BytesRead = fread(ReadBuffer, 1, sizeof ReadBuffer, stdin); - Writer.Write(ReadBuffer, BytesRead); - } while (!feof(stdin)); - - FileContents Contents; - Contents.Data.emplace_back(IoBuffer(IoBuffer::Clone, Writer.GetData(), Writer.GetSize())); - - return Contents; -} - -FileContents -ReadFile(std::filesystem::path Path) -{ - uint64_t FileSizeBytes; - void* Handle; - -#if ZEN_PLATFORM_WINDOWS - ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); - if (FromFile == INVALID_HANDLE_VALUE) - { - FromFile.Detach(); - return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())}; - } - - FILE_END_OF_FILE_INFO FileSize; - if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile)) - { - return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())}; - } - - FileSizeBytes = FileSize.EndOfFile.QuadPart; - Handle = FromFile.Detach(); -#else - int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd < 0) - { - FileContents Ret; - Ret.ErrorCode = std::error_code(zen::GetLastError(), std::system_category()); - return Ret; - } - - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - fstat(Fd, &Stat); - - FileSizeBytes = Stat.st_size; - Handle = (void*)uintptr_t(Fd); -#endif - - FileContents Contents; - Contents.Data.emplace_back(IoBuffer(IoBuffer::File, Handle, 0, FileSizeBytes)); - return Contents; -} - -bool -ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc) -{ -#if ZEN_PLATFORM_WINDOWS - ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); - if (FromFile == INVALID_HANDLE_VALUE) - { - FromFile.Detach(); - return false; - } - - std::vector<uint8_t> ReadBuffer(ChunkSize); - - for (;;) - { - DWORD dwBytesRead = 0; - BOOL Success = ::ReadFile(FromFile, ReadBuffer.data(), (DWORD)ReadBuffer.size(), &dwBytesRead, nullptr); - - if (!Success) - { - throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file scan failed"); - } - - if (dwBytesRead == 0) - break; - - ProcessFunc(ReadBuffer.data(), dwBytesRead); - } -#else - int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd < 0) - { - return false; - } - - bool Success = true; - - void* Buffer = malloc(ChunkSize); - while (true) - { - int BytesRead = read(Fd, Buffer, ChunkSize); - if (BytesRead < 0) - { - Success = false; - break; - } - - if (BytesRead == 0) - { - break; - } - - ProcessFunc(Buffer, BytesRead); - } - - free(Buffer); - close(Fd); - - if (!Success) - { - ThrowLastError("file scan failed"); - } -#endif // ZEN_PLATFORM_WINDOWS - - return true; -} - -void -PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out) -{ -#if ZEN_PLATFORM_WINDOWS - WideToUtf8(Path.native().c_str(), Out); -#else - Out << Path.c_str(); -#endif -} - -std::string -PathToUtf8(const std::filesystem::path& Path) -{ -#if ZEN_PLATFORM_WINDOWS - return WideToUtf8(Path.native().c_str()); -#else - return Path.string(); -#endif -} - -DiskSpace -DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error) -{ - using namespace std::filesystem; - - space_info SpaceInfo = space(Directory, Error); - if (Error) - { - return {}; - } - - return { - .Free = uint64_t(SpaceInfo.available), - .Total = uint64_t(SpaceInfo.capacity), - }; -} - -void -FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) -{ -#if ZEN_PLATFORM_WINDOWS - uint64_t FileInfoBuffer[8 * 1024]; - - FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; - bool Continue = true; - - CAtlFile RootDirHandle; - HRESULT hRes = - RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS); - - if (FAILED(hRes)) - { - ThrowSystemException(hRes, "Failed to open handle to volume root"); - } - - while (Continue) - { - BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer); - FibClass = FileIdBothDirectoryInfo; // Set up for next iteration - - uint64_t EntryOffset = 0; - - if (!Success) - { - DWORD LastError = GetLastError(); - - if (LastError == ERROR_NO_MORE_FILES) - { - break; - } - - throw std::system_error(std::error_code(LastError, std::system_category()), "file system traversal error"); - } - - for (;;) - { - const FILE_ID_BOTH_DIR_INFO* DirInfo = - reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset); - - std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t)); - - if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) - { - if (FileName == L"."sv || FileName == L".."sv) - { - // Not very interesting - } - else - { - const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName); - - if (ShouldDescend) - { - // Note that this recursion combined with the buffer could - // blow the stack, we should consider a different strategy - - std::filesystem::path FullPath = RootDir / FileName; - - TraverseFileSystem(FullPath, Visitor); - } - } - } - else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE) - { - ZEN_WARN("encountered device node during file system traversal: '{}' found in '{}'", WideToUtf8(FileName), RootDir); - } - else - { - Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart); - } - - const uint64_t NextOffset = DirInfo->NextEntryOffset; - - if (NextOffset == 0) - { - break; - } - - EntryOffset += NextOffset; - } - } -#else - /* Could also implement this using Linux's getdents() syscall */ - - DIR* Dir = opendir(RootDir.c_str()); - if (Dir == nullptr) - { - ThrowLastError(fmt::format("Failed to open directory for traversal: {}", RootDir.c_str())); - } - - for (struct dirent* Entry; (Entry = readdir(Dir));) - { - const char* FileName = Entry->d_name; - - struct stat Stat; - std::filesystem::path FullPath = RootDir / FileName; - stat(FullPath.c_str(), &Stat); - - if (S_ISDIR(Stat.st_mode)) - { - if (strcmp(FileName, ".") == 0 || strcmp(FileName, "..") == 0) - { - /* nop */ - } - else if (Visitor.VisitDirectory(RootDir, FileName)) - { - TraverseFileSystem(FullPath, Visitor); - } - } - else if (S_ISREG(Stat.st_mode)) - { - Visitor.VisitFile(RootDir, FileName, Stat.st_size); - } - else - { - ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}", - Stat.st_mode, - FileName, - RootDir.c_str()); - } - } - - closedir(Dir); -#endif // ZEN_PLATFORM_WINDOWS -} - -std::filesystem::path -PathFromHandle(void* NativeHandle) -{ -#if ZEN_PLATFORM_WINDOWS - if (NativeHandle == nullptr || NativeHandle == INVALID_HANDLE_VALUE) - { - return std::filesystem::path(); - } - - auto GetFinalPathNameByHandleWRetry = [](HANDLE hFile, LPWSTR lpszFilePath, DWORD cchFilePath, DWORD dwFlags) -> DWORD { - while (true) - { - DWORD Res = GetFinalPathNameByHandleW(hFile, lpszFilePath, cchFilePath, dwFlags); - if (Res == 0) - { - DWORD LastError = zen::GetLastError(); - // Under heavy concurrent loads we might get access denied on a file handle while trying to get path name. - // Retry if that is the case. - if (LastError != ERROR_ACCESS_DENIED) - { - ThrowSystemError(LastError, fmt::format("failed to get path from file handle {}", hFile)); - } - // Retry - continue; - } - ZEN_ASSERT(Res != 1); // We don't accept empty path names - return Res; - } - }; - - static const DWORD PathDataSize = 512; - wchar_t PathData[PathDataSize]; - DWORD RequiredLengthIncludingNul = GetFinalPathNameByHandleWRetry(NativeHandle, PathData, PathDataSize, FILE_NAME_OPENED); - if (RequiredLengthIncludingNul == 0) - { - ThrowLastError(fmt::format("failed to get path from file handle {}", NativeHandle)); - } - - if (RequiredLengthIncludingNul < PathDataSize) - { - std::wstring FullPath(PathData, gsl::narrow<size_t>(RequiredLengthIncludingNul)); - return FullPath; - } - - std::wstring FullPath; - FullPath.resize(RequiredLengthIncludingNul - 1); - - const DWORD FinalLength = GetFinalPathNameByHandleWRetry(NativeHandle, FullPath.data(), RequiredLengthIncludingNul, FILE_NAME_OPENED); - ZEN_UNUSED(FinalLength); - return FullPath; - -#elif ZEN_PLATFORM_LINUX - char Link[PATH_MAX]; - char Path[64]; - - sprintf(Path, "/proc/self/fd/%d", int(uintptr_t(NativeHandle))); - ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1); - if (BytesRead <= 0) - { - return std::filesystem::path(); - } - - Link[BytesRead] = '\0'; - return Link; -#elif ZEN_PLATFORM_MAC - int Fd = int(uintptr_t(NativeHandle)); - char Path[MAXPATHLEN]; - if (fcntl(Fd, F_GETPATH, Path) < 0) - { - return std::filesystem::path(); - } - - return Path; -#endif // ZEN_PLATFORM_WINDOWS -} - -std::filesystem::path -GetRunningExecutablePath() -{ -#if ZEN_PLATFORM_WINDOWS - TCHAR ExePath[MAX_PATH]; - DWORD PathLength = GetModuleFileName(NULL, ExePath, ZEN_ARRAY_COUNT(ExePath)); - - return {std::wstring_view(ExePath, PathLength)}; -#elif ZEN_PLATFORM_LINUX - char Link[256]; - ssize_t BytesRead = readlink("/proc/self/exe", Link, sizeof(Link) - 1); - if (BytesRead < 0) - return {}; - - Link[BytesRead] = '\0'; - return Link; -#elif ZEN_PLATFORM_MAC - char Buffer[PROC_PIDPATHINFO_MAXSIZE]; - - int SelfPid = GetCurrentProcessId(); - if (proc_pidpath(SelfPid, Buffer, sizeof(Buffer)) <= 0) - return {}; - - return Buffer; -#endif // ZEN_PLATFORM_WINDOWS -} - -void -MaximizeOpenFileCount() -{ -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - struct rlimit Limit; - int Error = getrlimit(RLIMIT_NOFILE, &Limit); - if (Error) - { - ZEN_WARN("failed getting rlimit RLIMIT_NOFILE, reason '{}'", zen::MakeErrorCode(Error).message()); - } - else - { - struct rlimit NewLimit = Limit; - NewLimit.rlim_cur = NewLimit.rlim_max; - ZEN_INFO("changing RLIMIT_NOFILE from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}", - Limit.rlim_cur, - Limit.rlim_max, - NewLimit.rlim_cur, - NewLimit.rlim_max); - - Error = setrlimit(RLIMIT_NOFILE, &NewLimit); - if (Error != 0) - { - ZEN_WARN("failed to set RLIMIT_NOFILE limits from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}, reason '{}'", - Limit.rlim_cur, - Limit.rlim_max, - NewLimit.rlim_cur, - NewLimit.rlim_max, - zen::MakeErrorCode(Error).message()); - } - } -#endif -} - -void -GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent) -{ - FileSystemTraversal Traversal; - struct Visitor : public FileSystemTraversal::TreeVisitor - { - Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {} - - virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent, - [[maybe_unused]] const path_view& File, - [[maybe_unused]] uint64_t FileSize) override - { - if (Flags & DirectoryContent::IncludeFilesFlag) - { - Content.Files.push_back(Parent / File); - } - } - - virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override - { - if (Flags & DirectoryContent::IncludeDirsFlag) - { - Content.Directories.push_back(Parent / DirectoryName); - } - return (Flags & DirectoryContent::RecursiveFlag) != 0; - } - - const uint8_t Flags; - DirectoryContent& Content; - } Visit(Flags, OutContent); - - Traversal.TraverseFileSystem(RootDir, Visit); -} - -std::string -GetEnvVariable(std::string_view VariableName) -{ - ZEN_ASSERT(!VariableName.empty()); -#if ZEN_PLATFORM_WINDOWS - - CHAR EnvVariableBuffer[1023 + 1]; - DWORD RESULT = GetEnvironmentVariableA(std::string(VariableName).c_str(), EnvVariableBuffer, sizeof(EnvVariableBuffer)); - if (RESULT > 0 && RESULT < sizeof(EnvVariableBuffer)) - { - return std::string(EnvVariableBuffer); - } -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - char* EnvVariable = getenv(std::string(VariableName).c_str()); - if (EnvVariable) - { - return std::string(EnvVariable); - } -#endif - return ""; -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -void -filesystem_forcelink() -{ -} - -TEST_CASE("filesystem") -{ - using namespace std::filesystem; - - // GetExePath -- this is not a great test as it's so dependent on where the this code gets linked in - path BinPath = GetRunningExecutablePath(); - const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver"; - CHECK(ExpectedExe); - CHECK(is_regular_file(BinPath)); - - // PathFromHandle - void* Handle; -# if ZEN_PLATFORM_WINDOWS - Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); - CHECK(Handle != INVALID_HANDLE_VALUE); -# else - int Fd = open(BinPath.c_str(), O_RDONLY | O_CLOEXEC); - CHECK(Fd >= 0); - Handle = (void*)uintptr_t(Fd); -# endif - - auto FromHandle = PathFromHandle((void*)uintptr_t(Handle)); - CHECK(equivalent(FromHandle, BinPath)); - -# if ZEN_PLATFORM_WINDOWS - CloseHandle(Handle); -# else - close(int(uintptr_t(Handle))); -# endif - - // Traversal - struct : public FileSystemTraversal::TreeVisitor - { - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override - { - bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected); - } - - virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } - - bool bFoundExpected = false; - std::filesystem::path Expected; - } Visitor; - Visitor.Expected = BinPath; - - FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor); - CHECK(Visitor.bFoundExpected); - - // Scan/read file - FileContents BinRead = ReadFile(BinPath); - std::vector<uint8_t> BinScan; - ScanFile(BinPath, 16 << 10, [&](const void* Data, size_t Size) { - const auto* Ptr = (uint8_t*)Data; - BinScan.insert(BinScan.end(), Ptr, Ptr + Size); - }); - CHECK_EQ(BinRead.Data.size(), 1); - CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize()); -} - -TEST_CASE("WriteFile") -{ - std::filesystem::path TempFile = GetRunningExecutablePath().parent_path(); - TempFile /= "write_file_test"; - - uint64_t Magics[] = { - 0x0'a9e'a9e'a9e'a9e'a9e, - 0x0'493'493'493'493'493, - }; - - struct - { - const void* Data; - size_t Size; - } MagicTests[] = { - { - Magics, - sizeof(Magics), - }, - { - Magics + 1, - sizeof(Magics[0]), - }, - }; - for (auto& MagicTest : MagicTests) - { - WriteFile(TempFile, IoBuffer(IoBuffer::Wrap, MagicTest.Data, MagicTest.Size)); - - FileContents MagicsReadback = ReadFile(TempFile); - CHECK_EQ(MagicsReadback.Data.size(), 1); - CHECK_EQ(MagicsReadback.Data[0].GetSize(), MagicTest.Size); - CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0); - } - - std::filesystem::remove(TempFile); -} - -TEST_CASE("DiskSpaceInfo") -{ - std::filesystem::path BinPath = GetRunningExecutablePath(); - - DiskSpace Space = {}; - - std::error_code Error; - Space = DiskSpaceInfo(BinPath, Error); - CHECK(!Error); - - bool Okay = DiskSpaceInfo(BinPath, Space); - CHECK(Okay); - - CHECK(int64_t(Space.Total) > 0); - CHECK(int64_t(Space.Free) > 0); // Hopefully there's at least one byte free -} - -TEST_CASE("PathBuilder") -{ -# if ZEN_PLATFORM_WINDOWS - const char* foo_bar = "/foo\\bar"; -# else - const char* foo_bar = "/foo/bar"; -# endif - - ExtendablePathBuilder<32> Path; - for (const char* Prefix : {"/foo", "/foo/"}) - { - Path.Reset(); - Path.Append(Prefix); - Path /= "bar"; - CHECK(Path.ToPath() == foo_bar); - } - - using fspath = std::filesystem::path; - - Path.Reset(); - Path.Append(fspath("/foo/")); - Path /= (fspath("bar")); - CHECK(Path.ToPath() == foo_bar); - -# if ZEN_PLATFORM_WINDOWS - Path.Reset(); - Path.Append(fspath(L"/\u0119oo/")); - Path /= L"bar"; - printf("%ls\n", Path.ToPath().c_str()); - CHECK(Path.ToView() == L"/\u0119oo/bar"); - CHECK(Path.ToPath() == L"\\\u0119oo\\bar"); -# endif -} - -#endif - -} // namespace zen diff --git a/zencore/include/zencore/atomic.h b/zencore/include/zencore/atomic.h deleted file mode 100644 index bf549e21d..000000000 --- a/zencore/include/zencore/atomic.h +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -#if ZEN_COMPILER_MSC -# include <intrin.h> -#else -# include <atomic> -#endif - -#include <cinttypes> - -namespace zen { - -inline uint32_t -AtomicIncrement(volatile uint32_t& value) -{ -#if ZEN_COMPILER_MSC - return _InterlockedIncrement((long volatile*)&value); -#else - return ((std::atomic<uint32_t>*)(&value))->fetch_add(1, std::memory_order_seq_cst) + 1; -#endif -} -inline uint32_t -AtomicDecrement(volatile uint32_t& value) -{ -#if ZEN_COMPILER_MSC - return _InterlockedDecrement((long volatile*)&value); -#else - return ((std::atomic<uint32_t>*)(&value))->fetch_sub(1, std::memory_order_seq_cst) - 1; -#endif -} - -inline uint64_t -AtomicIncrement(volatile uint64_t& value) -{ -#if ZEN_COMPILER_MSC - return _InterlockedIncrement64((__int64 volatile*)&value); -#else - return ((std::atomic<uint64_t>*)(&value))->fetch_add(1, std::memory_order_seq_cst) + 1; -#endif -} -inline uint64_t -AtomicDecrement(volatile uint64_t& value) -{ -#if ZEN_COMPILER_MSC - return _InterlockedDecrement64((__int64 volatile*)&value); -#else - return ((std::atomic<uint64_t>*)(&value))->fetch_sub(1, std::memory_order_seq_cst) - 1; -#endif -} - -inline uint32_t -AtomicAdd(volatile uint32_t& value, uint32_t amount) -{ -#if ZEN_COMPILER_MSC - return _InterlockedExchangeAdd((long volatile*)&value, amount); -#else - return ((std::atomic<uint32_t>*)(&value))->fetch_add(amount, std::memory_order_seq_cst); -#endif -} -inline uint64_t -AtomicAdd(volatile uint64_t& value, uint64_t amount) -{ -#if ZEN_COMPILER_MSC - return _InterlockedExchangeAdd64((__int64 volatile*)&value, amount); -#else - return ((std::atomic<uint64_t>*)(&value))->fetch_add(amount, std::memory_order_seq_cst); -#endif -} - -} // namespace zen diff --git a/zencore/include/zencore/base64.h b/zencore/include/zencore/base64.h deleted file mode 100644 index 4d78b085f..000000000 --- a/zencore/include/zencore/base64.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -namespace zen { - -struct Base64 -{ - template<typename CharType> - static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest); - - static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; } -}; - -} // namespace zen diff --git a/zencore/include/zencore/blake3.h b/zencore/include/zencore/blake3.h deleted file mode 100644 index b31b710a7..000000000 --- a/zencore/include/zencore/blake3.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <cinttypes> -#include <compare> -#include <cstring> - -#include <zencore/memory.h> - -namespace zen { - -class CompositeBuffer; -class StringBuilderBase; - -/** - * BLAKE3 hash - 256 bits - */ -struct BLAKE3 -{ - uint8_t Hash[32]; - - inline auto operator<=>(const BLAKE3& Rhs) const = default; - - static BLAKE3 HashBuffer(const CompositeBuffer& Buffer); - static BLAKE3 HashMemory(const void* Data, size_t ByteCount); - static BLAKE3 FromHexString(const char* String); - const char* ToHexString(char* OutString /* 40 characters + NUL terminator */) const; - StringBuilderBase& ToHexString(StringBuilderBase& OutBuilder) const; - - static const int StringLength = 64; - typedef char String_t[StringLength + 1]; - - static BLAKE3 Zero; // Initialized to all zeroes - - struct Hasher - { - size_t operator()(const BLAKE3& v) const - { - size_t h; - memcpy(&h, v.Hash, sizeof h); - return h; - } - }; -}; - -struct BLAKE3Stream -{ - BLAKE3Stream(); - - void Reset(); // Begin streaming hash compute (not needed on freshly constructed instance) - BLAKE3Stream& Append(const void* data, size_t byteCount); // Append another chunk - BLAKE3Stream& Append(MemoryView DataView) { return Append(DataView.GetData(), DataView.GetSize()); } // Append another chunk - BLAKE3 GetHash(); // Obtain final hash. If you wish to reuse the instance call reset() - -private: - alignas(16) uint8_t m_HashState[2048]; -}; - -void blake3_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/blockingqueue.h b/zencore/include/zencore/blockingqueue.h deleted file mode 100644 index f92df5a54..000000000 --- a/zencore/include/zencore/blockingqueue.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <atomic> -#include <condition_variable> -#include <deque> -#include <mutex> - -namespace zen { - -template<typename T> -class BlockingQueue -{ -public: - BlockingQueue() = default; - - ~BlockingQueue() { CompleteAdding(); } - - void Enqueue(T&& Item) - { - { - std::lock_guard Lock(m_Lock); - m_Queue.emplace_back(std::move(Item)); - m_Size++; - } - - m_NewItemSignal.notify_one(); - } - - bool WaitAndDequeue(T& Item) - { - if (m_CompleteAdding.load()) - { - return false; - } - - std::unique_lock Lock(m_Lock); - m_NewItemSignal.wait(Lock, [this]() { return !m_Queue.empty() || m_CompleteAdding.load(); }); - - if (!m_Queue.empty()) - { - Item = std::move(m_Queue.front()); - m_Queue.pop_front(); - m_Size--; - - return true; - } - - return false; - } - - void CompleteAdding() - { - if (!m_CompleteAdding.load()) - { - m_CompleteAdding.store(true); - m_NewItemSignal.notify_all(); - } - } - - std::size_t Size() const - { - std::unique_lock Lock(m_Lock); - return m_Queue.size(); - } - -private: - mutable std::mutex m_Lock; - std::condition_variable m_NewItemSignal; - std::deque<T> m_Queue; - std::atomic_bool m_CompleteAdding{false}; - std::atomic_uint32_t m_Size; -}; - -} // namespace zen diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h deleted file mode 100644 index b546f97aa..000000000 --- a/zencore/include/zencore/compactbinary.h +++ /dev/null @@ -1,1475 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -#include <zencore/enumflags.h> -#include <zencore/intmath.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/memory.h> -#include <zencore/meta.h> -#include <zencore/sharedbuffer.h> -#include <zencore/uid.h> -#include <zencore/varint.h> - -#include <functional> -#include <memory> -#include <string> -#include <string_view> -#include <type_traits> -#include <vector> - -#include <gsl/gsl-lite.hpp> - -namespace zen { - -class CbObjectView; -class CbArrayView; -class BinaryReader; -class BinaryWriter; -class CompressedBuffer; -class CbValue; - -class DateTime -{ -public: - explicit DateTime(uint64_t InTicks) : Ticks(InTicks) {} - inline DateTime(int Year, int Month, int Day, int Hours = 0, int Minutes = 0, int Seconds = 0, int MilliSeconds = 0) - { - Set(Year, Month, Day, Hours, Minutes, Seconds, MilliSeconds); - } - - inline uint64_t GetTicks() const { return Ticks; } - - static uint64_t NowTicks(); - static DateTime Now(); - - int GetYear() const; - int GetMonth() const; - int GetDay() const; - int GetHour() const; - int GetHour12() const; - int GetMinute() const; - int GetSecond() const; - int GetMillisecond() const; - void GetDate(int& Year, int& Month, int& Day) const; - - inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; } - inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; } - - std::string ToString(const char* Format) const; - std::string ToIso8601() const; - -private: - void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond); - uint64_t Ticks; // 1 tick == 0.1us == 100ns, epoch == Jan 1st 0001 -}; - -class TimeSpan -{ -public: - explicit TimeSpan(uint64_t InTicks) : Ticks(InTicks) {} - inline TimeSpan(int Hours, int Minutes, int Seconds) { Set(0, Hours, Minutes, Seconds, 0); } - inline TimeSpan(int Days, int Hours, int Minutes, int Seconds) { Set(Days, Hours, Minutes, Seconds, 0); } - inline TimeSpan(int Days, int Hours, int Minutes, int Seconds, int Nanos) { Set(Days, Hours, Minutes, Seconds, Nanos); } - - inline uint64_t GetTicks() const { return Ticks; } - inline bool operator==(const TimeSpan& Rhs) const { return Ticks == Rhs.Ticks; } - inline auto operator<=>(const TimeSpan& Rhs) const { return Ticks - Rhs.Ticks; } - - /** - * Time span related constants. - */ - - /** The maximum number of ticks that can be represented in FTimespan. */ - static constexpr int64_t MaxTicks = 9223372036854775807; - - /** The minimum number of ticks that can be represented in FTimespan. */ - static constexpr int64_t MinTicks = -9223372036854775807 - 1; - - /** The number of nanoseconds per tick. */ - static constexpr int64_t NanosecondsPerTick = 100; - - /** The number of timespan ticks per day. */ - static constexpr int64_t TicksPerDay = 864000000000; - - /** The number of timespan ticks per hour. */ - static constexpr int64_t TicksPerHour = 36000000000; - - /** The number of timespan ticks per microsecond. */ - static constexpr int64_t TicksPerMicrosecond = 10; - - /** The number of timespan ticks per millisecond. */ - static constexpr int64_t TicksPerMillisecond = 10000; - - /** The number of timespan ticks per minute. */ - static constexpr int64_t TicksPerMinute = 600000000; - - /** The number of timespan ticks per second. */ - static constexpr int64_t TicksPerSecond = 10000000; - - /** The number of timespan ticks per week. */ - static constexpr int64_t TicksPerWeek = 6048000000000; - - /** The number of timespan ticks per year (365 days, not accounting for leap years). */ - static constexpr int64_t TicksPerYear = 365 * TicksPerDay; - - int GetFractionTicks() const { return (int)(Ticks % TicksPerSecond); } - - int GetFractionMicro() const { return (int)((Ticks % TicksPerSecond) / TicksPerMicrosecond); } - - int GetFractionMilli() const { return (int)((Ticks % TicksPerSecond) / TicksPerMillisecond); } - - int GetFractionNano() const { return (int)((Ticks % TicksPerSecond) * NanosecondsPerTick); } - - int GetDays() const { return (int)(Ticks / TicksPerDay); } - - int GetHours() const { return (int)((Ticks / TicksPerHour) % 24); } - - int GetMinutes() const { return (int)((Ticks / TicksPerMinute) % 60); } - - int GetSeconds() const { return (int)((Ticks / TicksPerSecond) % 60); } - - ZENCORE_API std::string ToString(const char* Format) const; - ZENCORE_API std::string ToString() const; - -private: - void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano); - - uint64_t Ticks; -}; - -struct Guid -{ - uint32_t A, B, C, D; - - StringBuilderBase& ToString(StringBuilderBase& OutString) const; -}; - -////////////////////////////////////////////////////////////////////////// - -/** - * Field types and flags for CbField. - * - * This is a private type and is only declared here to enable inline use below. - * - * DO NOT CHANGE THE VALUE OF ANY MEMBERS OF THIS ENUM! - * BACKWARD COMPATIBILITY REQUIRES THAT THESE VALUES BE FIXED! - * SERIALIZATION USES HARD-CODED CONSTANTS BASED ON THESE VALUES! - */ -enum class CbFieldType : uint8_t -{ - /** A field type that does not occur in a valid object. */ - None = 0x00, - - /** Null. Payload is empty. */ - Null = 0x01, - - /** - * Object is an array of fields with unique non-empty names. - * - * Payload is a VarUInt byte count for the encoded fields followed by the fields. - */ - Object = 0x02, - /** - * UniformObject is an array of fields with the same field types and unique non-empty names. - * - * Payload is a VarUInt byte count for the encoded fields followed by the fields. - */ - UniformObject = 0x03, - - /** - * Array is an array of fields with no name that may be of different types. - * - * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by the fields. - */ - Array = 0x04, - /** - * UniformArray is an array of fields with no name and with the same field type. - * - * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by field type, - * followed by the fields without their field type. - */ - UniformArray = 0x05, - - /** Binary. Payload is a VarUInt byte count followed by the data. */ - Binary = 0x06, - - /** String in UTF-8. Payload is a VarUInt byte count then an unterminated UTF-8 string. */ - String = 0x07, - - /** - * Non-negative integer with the range of a 64-bit unsigned integer. - * - * Payload is the value encoded as a VarUInt. - */ - IntegerPositive = 0x08, - /** - * Negative integer with the range of a 64-bit signed integer. - * - * Payload is the ones' complement of the value encoded as a VarUInt. - */ - IntegerNegative = 0x09, - - /** Single precision float. Payload is one big endian IEEE 754 binary32 float. */ - Float32 = 0x0a, - /** Double precision float. Payload is one big endian IEEE 754 binary64 float. */ - Float64 = 0x0b, - - /** Boolean false value. Payload is empty. */ - BoolFalse = 0x0c, - /** Boolean true value. Payload is empty. */ - BoolTrue = 0x0d, - - /** - * ObjectAttachment is a reference to a compact binary attachment stored externally. - * - * Payload is a 160-bit hash digest of the referenced compact binary data. - */ - ObjectAttachment = 0x0e, - /** - * BinaryAttachment is a reference to a binary attachment stored externally. - * - * Payload is a 160-bit hash digest of the referenced binary data. - */ - BinaryAttachment = 0x0f, - - /** Hash. Payload is a 160-bit hash digest. */ - Hash = 0x10, - /** UUID/GUID. Payload is a 128-bit UUID as defined by RFC 4122. */ - Uuid = 0x11, - - /** - * Date and time between 0001-01-01 00:00:00.0000000 and 9999-12-31 23:59:59.9999999. - * - * Payload is a big endian int64 count of 100ns ticks since 0001-01-01 00:00:00.0000000. - */ - DateTime = 0x12, - /** - * Difference between two date/time values. - * - * Payload is a big endian int64 count of 100ns ticks in the span, and may be negative. - */ - TimeSpan = 0x13, - - /** - * Object ID - * - * Payload is a 12-byte opaque identifier - */ - ObjectId = 0x14, - - /** - * CustomById identifies the sub-type of its payload by an integer identifier. - * - * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed - * by a VarUInt of the sub-type identifier then the payload of the sub-type. - */ - CustomById = 0x1e, - /** - * CustomByType identifies the sub-type of its payload by a string identifier. - * - * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed - * by a VarUInt byte count of the unterminated sub-type identifier, then the sub-type identifier - * without termination, then the payload of the sub-type. - */ - CustomByName = 0x1f, - - /** Reserved for future use as a flag. Do not add types in this range. */ - Reserved = 0x20, - - /** - * A transient flag which indicates that the object or array containing this field has stored - * the field type before the payload and name. Non-uniform objects and fields will set this. - * - * Note: Since the flag must never be serialized, this bit may be repurposed in the future. - */ - HasFieldType = 0x40, - - /** A persisted flag which indicates that the field has a name stored before the payload. */ - HasFieldName = 0x80, -}; - -ENUM_CLASS_FLAGS(CbFieldType); - -/** Functions that operate on CbFieldType. */ -class CbFieldTypeOps -{ - static constexpr CbFieldType SerializedTypeMask = CbFieldType(0b1011'1111); - static constexpr CbFieldType TypeMask = CbFieldType(0b0011'1111); - static constexpr CbFieldType ObjectMask = CbFieldType(0b0011'1110); - static constexpr CbFieldType ObjectBase = CbFieldType(0b0000'0010); - static constexpr CbFieldType ArrayMask = CbFieldType(0b0011'1110); - static constexpr CbFieldType ArrayBase = CbFieldType(0b0000'0100); - static constexpr CbFieldType IntegerMask = CbFieldType(0b0011'1110); - static constexpr CbFieldType IntegerBase = CbFieldType(0b0000'1000); - static constexpr CbFieldType FloatMask = CbFieldType(0b0011'1100); - static constexpr CbFieldType FloatBase = CbFieldType(0b0000'1000); - static constexpr CbFieldType BoolMask = CbFieldType(0b0011'1110); - static constexpr CbFieldType BoolBase = CbFieldType(0b0000'1100); - static constexpr CbFieldType AttachmentMask = CbFieldType(0b0011'1110); - static constexpr CbFieldType AttachmentBase = CbFieldType(0b0000'1110); - - static void StaticAssertTypeConstants(); - -public: - /** The type with flags removed. */ - static constexpr inline CbFieldType GetType(CbFieldType Type) { return Type & TypeMask; } - /** The type with transient flags removed. */ - static constexpr inline CbFieldType GetSerializedType(CbFieldType Type) { return Type & SerializedTypeMask; } - - static constexpr inline bool HasFieldType(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldType); } - static constexpr inline bool HasFieldName(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldName); } - - static constexpr inline bool IsNone(CbFieldType Type) { return GetType(Type) == CbFieldType::None; } - static constexpr inline bool IsNull(CbFieldType Type) { return GetType(Type) == CbFieldType::Null; } - - static constexpr inline bool IsObject(CbFieldType Type) { return (Type & ObjectMask) == ObjectBase; } - static constexpr inline bool IsArray(CbFieldType Type) { return (Type & ArrayMask) == ArrayBase; } - - static constexpr inline bool IsBinary(CbFieldType Type) { return GetType(Type) == CbFieldType::Binary; } - static constexpr inline bool IsString(CbFieldType Type) { return GetType(Type) == CbFieldType::String; } - - static constexpr inline bool IsInteger(CbFieldType Type) { return (Type & IntegerMask) == IntegerBase; } - /** Whether the field is a float, or integer due to implicit conversion. */ - static constexpr inline bool IsFloat(CbFieldType Type) { return (Type & FloatMask) == FloatBase; } - static constexpr inline bool IsBool(CbFieldType Type) { return (Type & BoolMask) == BoolBase; } - - static constexpr inline bool IsObjectAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectAttachment; } - static constexpr inline bool IsBinaryAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::BinaryAttachment; } - static constexpr inline bool IsAttachment(CbFieldType Type) { return (Type & AttachmentMask) == AttachmentBase; } - - static constexpr inline bool IsHash(CbFieldType Type) - { - switch (GetType(Type)) - { - case CbFieldType::Hash: - case CbFieldType::BinaryAttachment: - case CbFieldType::ObjectAttachment: - return true; - default: - return false; - } - } - - static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; } - static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; } - - static constexpr inline bool IsCustomById(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomById; } - static constexpr inline bool IsCustomByName(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomByName; } - - static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; } - static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; } - - /** Whether the type is or may contain fields of any attachment type. */ - static constexpr inline bool MayContainAttachments(CbFieldType Type) - { - return int(IsObject(Type) == true) | int(IsArray(Type) == true) | int(IsAttachment(Type) == true); - } -}; - -/** Errors that can occur when accessing a field. */ -enum class CbFieldError : uint8_t -{ - /** The field is not in an error state. */ - None, - /** The value type does not match the requested type. */ - TypeError, - /** The value is out of range for the requested type. */ - RangeError, -}; - -class ICbVisitor -{ -public: - virtual void SetName(std::string_view Name) = 0; - virtual void BeginObject() = 0; - virtual void EndObject() = 0; - virtual void BeginArray() = 0; - virtual void EndArray() = 0; - virtual void VisitNull() = 0; - virtual void VisitBinary(SharedBuffer Value) = 0; - virtual void VisitString(std::string_view Value) = 0; - virtual void VisitInteger(int64_t Value) = 0; - virtual void VisitInteger(uint64_t Value) = 0; - virtual void VisitFloat(float Value) = 0; - virtual void VisitDouble(double Value) = 0; - virtual void VisitBool(bool value) = 0; - virtual void VisitCbAttachment(const IoHash& Value) = 0; - virtual void VisitBinaryAttachment(const IoHash& Value) = 0; - virtual void VisitHash(const IoHash& Value) = 0; - virtual void VisitUuid(const Guid& Value) = 0; - virtual void VisitObjectId(const Oid& Value) = 0; - virtual void VisitDateTime(DateTime Value) = 0; - virtual void VisitTimeSpan(TimeSpan Value) = 0; -}; - -/** A custom compact binary field type with an integer identifier. */ -struct CbCustomById -{ - /** An identifier for the sub-type of the field. */ - uint64_t Id = 0; - /** A view of the value. Lifetime is tied to the field that the value is associated with. */ - MemoryView Data; -}; - -/** A custom compact binary field type with a string identifier. */ -struct CbCustomByName -{ - /** An identifier for the sub-type of the field. Lifetime is tied to the field that the name is associated with. */ - std::u8string_view Name; - /** A view of the value. Lifetime is tied to the field that the value is associated with. */ - MemoryView Data; -}; - -namespace CompactBinaryPrivate { - /** Parameters for converting to an integer. */ - struct IntegerParams - { - /** Whether the output type has a sign bit. */ - uint32_t IsSigned : 1; - /** Bits of magnitude. (7 for int8) */ - uint32_t MagnitudeBits : 31; - }; - - /** Make integer params for the given integer type. */ - template<typename IntType> - static constexpr inline IntegerParams MakeIntegerParams() - { - IntegerParams Params; - Params.IsSigned = IntType(-1) < IntType(0); - Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned; - return Params; - } - -} // namespace CompactBinaryPrivate - -/** - * An atom of data in the compact binary format. - * - * Accessing the value of a field is always a safe operation, even if accessed as the wrong type. - * An invalid access will return a default value for the requested type, and set an error code on - * the field that can be checked with GetLastError and HasLastError. A valid access will clear an - * error from a previous invalid access. - * - * A field is encoded in one or more bytes, depending on its type and the type of object or array - * that contains it. A field of an object or array which is non-uniform encodes its field type in - * the first byte, and includes the HasFieldName flag for a field in an object. The field name is - * encoded in a variable-length unsigned integer of its size in bytes, for named fields, followed - * by that many bytes of the UTF-8 encoding of the name with no null terminator. The remainder of - * the field is the payload and is described in the field type enum. Every field must be uniquely - * addressable when encoded, which means a zero-byte field is not permitted, and only arises in a - * uniform array of fields with no payload, where the answer is to encode as a non-uniform array. - * - * This type only provides a view into memory and does not perform any memory management itself. - * Use CbFieldRef to hold a reference to the underlying memory when necessary. - */ - -class CbFieldView -{ -public: - CbFieldView() = default; - - ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType); - - /** Construct a field from a value, without access to the name. */ - inline explicit CbFieldView(const CbValue& Value); - - /** Returns the name of the field if it has a name, otherwise an empty view. */ - constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); } - /** Returns the name of the field if it has a name, otherwise an empty view. */ - constexpr inline std::u8string_view GetU8Name() const - { - return std::u8string_view(static_cast<const char8_t*>(Payload) - NameLen, NameLen); - } - - /** Returns the value for unchecked access. Prefer the typed accessors below. */ - inline CbValue GetValue() const; - - ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView()); - ZENCORE_API CbObjectView AsObjectView(); - ZENCORE_API CbArrayView AsArrayView(); - ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view()); - ZENCORE_API std::u8string_view AsU8String(std::u8string_view Default = std::u8string_view()); - - ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const; - - /** Access the field as an int8. Returns the provided default on error. */ - inline int8_t AsInt8(int8_t Default = 0) { return AsInteger<int8_t>(Default); } - /** Access the field as an int16. Returns the provided default on error. */ - inline int16_t AsInt16(int16_t Default = 0) { return AsInteger<int16_t>(Default); } - /** Access the field as an int32. Returns the provided default on error. */ - inline int32_t AsInt32(int32_t Default = 0) { return AsInteger<int32_t>(Default); } - /** Access the field as an int64. Returns the provided default on error. */ - inline int64_t AsInt64(int64_t Default = 0) { return AsInteger<int64_t>(Default); } - /** Access the field as a uint8. Returns the provided default on error. */ - inline uint8_t AsUInt8(uint8_t Default = 0) { return AsInteger<uint8_t>(Default); } - /** Access the field as a uint16. Returns the provided default on error. */ - inline uint16_t AsUInt16(uint16_t Default = 0) { return AsInteger<uint16_t>(Default); } - /** Access the field as a uint32. Returns the provided default on error. */ - inline uint32_t AsUInt32(uint32_t Default = 0) { return AsInteger<uint32_t>(Default); } - /** Access the field as a uint64. Returns the provided default on error. */ - inline uint64_t AsUInt64(uint64_t Default = 0) { return AsInteger<uint64_t>(Default); } - - /** Access the field as a float. Returns the provided default on error. */ - ZENCORE_API float AsFloat(float Default = 0.0f); - /** Access the field as a double. Returns the provided default on error. */ - ZENCORE_API double AsDouble(double Default = 0.0); - - /** Access the field as a bool. Returns the provided default on error. */ - ZENCORE_API bool AsBool(bool bDefault = false); - - /** Access the field as a hash referencing a compact binary attachment. Returns the provided default on error. */ - ZENCORE_API IoHash AsObjectAttachment(const IoHash& Default = IoHash()); - /** Access the field as a hash referencing a binary attachment. Returns the provided default on error. */ - ZENCORE_API IoHash AsBinaryAttachment(const IoHash& Default = IoHash()); - /** Access the field as a hash referencing an attachment. Returns the provided default on error. */ - ZENCORE_API IoHash AsAttachment(const IoHash& Default = IoHash()); - - /** Access the field as a hash. Returns the provided default on error. */ - ZENCORE_API IoHash AsHash(const IoHash& Default = IoHash()); - - /** Access the field as a UUID. Returns a nil UUID on error. */ - ZENCORE_API Guid AsUuid(); - /** Access the field as a UUID. Returns the provided default on error. */ - ZENCORE_API Guid AsUuid(const Guid& Default); - - /** Access the field as an OID. Returns a nil OID on error. */ - ZENCORE_API Oid AsObjectId(); - /** Access the field as a OID. Returns the provided default on error. */ - ZENCORE_API Oid AsObjectId(const Oid& Default); - - /** Access the field as a custom sub-type with an integer identifier. Returns the provided default on error. */ - ZENCORE_API CbCustomById AsCustomById(CbCustomById Default); - /** Access the field as a custom sub-type with a string identifier. Returns the provided default on error. */ - ZENCORE_API CbCustomByName AsCustomByName(CbCustomByName Default); - - /** Access the field as a date/time tick count. Returns the provided default on error. */ - ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0); - - /** Access the field as a date/time. Returns a date/time at the epoch on error. */ - ZENCORE_API DateTime AsDateTime(); - /** Access the field as a date/time. Returns the provided default on error. */ - ZENCORE_API DateTime AsDateTime(DateTime Default); - - /** Access the field as a timespan tick count. Returns the provided default on error. */ - ZENCORE_API int64_t AsTimeSpanTicks(int64_t Default = 0); - - /** Access the field as a timespan. Returns an empty timespan on error. */ - ZENCORE_API TimeSpan AsTimeSpan(); - /** Access the field as a timespan. Returns the provided default on error. */ - ZENCORE_API TimeSpan AsTimeSpan(TimeSpan Default); - - /** True if the field has a name. */ - constexpr inline bool HasName() const { return CbFieldTypeOps::HasFieldName(Type); } - - constexpr inline bool IsNull() const { return CbFieldTypeOps::IsNull(Type); } - - constexpr inline bool IsObject() const { return CbFieldTypeOps::IsObject(Type); } - constexpr inline bool IsArray() const { return CbFieldTypeOps::IsArray(Type); } - - constexpr inline bool IsBinary() const { return CbFieldTypeOps::IsBinary(Type); } - constexpr inline bool IsString() const { return CbFieldTypeOps::IsString(Type); } - - /** Whether the field is an integer of unspecified range and sign. */ - constexpr inline bool IsInteger() const { return CbFieldTypeOps::IsInteger(Type); } - /** Whether the field is a float, or integer that supports implicit conversion. */ - constexpr inline bool IsFloat() const { return CbFieldTypeOps::IsFloat(Type); } - constexpr inline bool IsBool() const { return CbFieldTypeOps::IsBool(Type); } - - constexpr inline bool IsObjectAttachment() const { return CbFieldTypeOps::IsObjectAttachment(Type); } - constexpr inline bool IsBinaryAttachment() const { return CbFieldTypeOps::IsBinaryAttachment(Type); } - constexpr inline bool IsAttachment() const { return CbFieldTypeOps::IsAttachment(Type); } - - constexpr inline bool IsHash() const { return CbFieldTypeOps::IsHash(Type); } - constexpr inline bool IsUuid() const { return CbFieldTypeOps::IsUuid(Type); } - constexpr inline bool IsObjectId() const { return CbFieldTypeOps::IsObjectId(Type); } - - constexpr inline bool IsDateTime() const { return CbFieldTypeOps::IsDateTime(Type); } - constexpr inline bool IsTimeSpan() const { return CbFieldTypeOps::IsTimeSpan(Type); } - - /** Whether the field has a value. */ - constexpr inline explicit operator bool() const { return HasValue(); } - - /** - * Whether the field has a value. - * - * All fields in a valid object or array have a value. A field with no value is returned when - * finding a field by name fails or when accessing an iterator past the end. - */ - constexpr inline bool HasValue() const { return !CbFieldTypeOps::IsNone(Type); }; - - /** Whether the last field access encountered an error. */ - constexpr inline bool HasError() const { return Error != CbFieldError::None; } - - /** The type of error that occurred on the last field access, or None. */ - constexpr inline CbFieldError GetError() const { return Error; } - - /** Returns the size of the field in bytes, including the type and name. */ - ZENCORE_API uint64_t GetSize() const; - - /** Calculate the hash of the field, including the type and name. */ - ZENCORE_API IoHash GetHash() const; - - ZENCORE_API void GetHash(IoHashStream& HashStream) const; - - /** Feed the field (including type and name) to the stream function */ - inline void WriteToStream(auto Hash) const - { - const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type); - Hash(&SerializedType, sizeof(SerializedType)); - auto View = GetViewNoType(); - Hash(View.GetData(), View.GetSize()); - } - - /** Copy the field into a buffer of exactly GetSize() bytes, including the type and name. */ - ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; - - /** Copy the field into an archive, including its type and name. */ - ZENCORE_API void CopyTo(BinaryWriter& Ar) const; - - /** - * Whether this field is identical to the other field. - * - * Performs a deep comparison of any contained arrays or objects and their fields. Comparison - * assumes that both fields are valid and are written in the canonical format. Fields must be - * written in the same order in arrays and objects, and name comparison is case sensitive. If - * these assumptions do not hold, this may return false for equivalent inputs. Validation can - * be performed with ValidateCompactBinary, except for field order and field name case. - */ - ZENCORE_API bool Equals(const CbFieldView& Other) const; - - /** Returns a view of the field, including the type and name when present. */ - ZENCORE_API MemoryView GetView() const; - - /** - * Try to get a view of the field as it would be serialized, such as by CopyTo. - * - * A serialized view is not available if the field has an externally-provided type. - * Access the serialized form of such fields using CopyTo or FCbFieldRef::Clone. - */ - inline bool TryGetSerializedView(MemoryView& OutView) const - { - if (CbFieldTypeOps::HasFieldType(Type)) - { - OutView = GetView(); - return true; - } - return false; - } - -protected: - /** Returns a view of the name and value payload, which excludes the type. */ - ZENCORE_API MemoryView GetViewNoType() const; - - /** Returns a view of the value payload, which excludes the type and name. */ - inline MemoryView GetPayloadView() const { return MemoryView(Payload, GetPayloadSize()); } - - /** Returns the type of the field including flags. */ - constexpr inline CbFieldType GetType() const { return Type; } - - /** Returns the start of the value payload. */ - constexpr inline const void* GetPayload() const { return Payload; } - - /** Returns the end of the value payload. */ - inline const void* GetPayloadEnd() const { return static_cast<const uint8_t*>(Payload) + GetPayloadSize(); } - - /** Returns the size of the value payload in bytes, which is the field excluding the type and name. */ - ZENCORE_API uint64_t GetPayloadSize() const; - - /** Assign a field from a pointer to its data and an optional externally-provided type. */ - inline void Assign(const void* InData, const CbFieldType InType) - { - static_assert(std::is_trivially_destructible<CbFieldView>::value, - "This optimization requires CbField to be trivially destructible!"); - new (this) CbFieldView(InData, InType); - } - -private: - /** - * Access the field as the given integer type. - * - * Returns the provided default if the value cannot be represented in the output type. - */ - template<typename IntType> - inline IntType AsInteger(IntType Default) - { - return IntType(AsInteger(uint64_t(Default), CompactBinaryPrivate::MakeIntegerParams<IntType>())); - } - - ZENCORE_API uint64_t AsInteger(uint64_t Default, CompactBinaryPrivate::IntegerParams Params); - -private: - /** The field type, with the transient HasFieldType flag if the field contains its type. */ - CbFieldType Type = CbFieldType::None; - /** The error (if any) that occurred on the last field access. */ - CbFieldError Error = CbFieldError::None; - /** The number of bytes for the name stored before the payload. */ - uint32_t NameLen = 0; - /** The value payload, which also points to the end of the name. */ - const void* Payload = nullptr; -}; - -template<typename FieldType> -class TCbFieldIterator : public FieldType -{ -public: - /** Construct an empty field range. */ - constexpr TCbFieldIterator() = default; - - inline TCbFieldIterator& operator++() - { - const void* const PayloadEnd = FieldType::GetPayloadEnd(); - const int64_t AtEndMask = int64_t(PayloadEnd == FieldsEnd) - 1; - const CbFieldType NextType = CbFieldType(int64_t(FieldType::GetType()) & AtEndMask); - const void* const NextField = reinterpret_cast<const void*>(int64_t(PayloadEnd) & AtEndMask); - const void* const NextFieldsEnd = reinterpret_cast<const void*>(int64_t(FieldsEnd) & AtEndMask); - - FieldType::Assign(NextField, NextType); - FieldsEnd = NextFieldsEnd; - return *this; - } - - inline TCbFieldIterator operator++(int) - { - TCbFieldIterator It(*this); - ++*this; - return It; - } - - constexpr inline FieldType& operator*() { return *this; } - constexpr inline FieldType* operator->() { return this; } - - /** Reset this to an empty field range. */ - inline void Reset() { *this = TCbFieldIterator(); } - - /** Returns the size of the fields in the range in bytes. */ - ZENCORE_API uint64_t GetRangeSize() const; - - /** Calculate the hash of every field in the range. */ - ZENCORE_API IoHash GetRangeHash() const; - ZENCORE_API void GetRangeHash(IoHashStream& Hash) const; - - using FieldType::Equals; - - template<typename OtherFieldType> - constexpr inline bool Equals(const TCbFieldIterator<OtherFieldType>& Other) const - { - return FieldType::GetPayload() == Other.OtherFieldType::GetPayload() && FieldsEnd == Other.FieldsEnd; - } - - template<typename OtherFieldType> - constexpr inline bool operator==(const TCbFieldIterator<OtherFieldType>& Other) const - { - return Equals(Other); - } - - template<typename OtherFieldType> - constexpr inline bool operator!=(const TCbFieldIterator<OtherFieldType>& Other) const - { - return !Equals(Other); - } - - /** Copy the field range into a buffer of exactly GetRangeSize() bytes. */ - ZENCORE_API void CopyRangeTo(MutableMemoryView Buffer) const; - - /** Invoke the visitor for every attachment in the field range. */ - ZENCORE_API void IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const; - - /** Create a view of every field in the range. */ - inline MemoryView GetRangeView() const { return MemoryView(FieldType::GetView().GetData(), FieldsEnd); } - - /** - * Try to get a view of every field in the range as they would be serialized. - * - * A serialized view is not available if the underlying fields have an externally-provided type. - * Access the serialized form of such ranges using CbFieldRefIterator::CloneRange. - */ - inline bool TryGetSerializedRangeView(MemoryView& OutView) const - { - if (CbFieldTypeOps::HasFieldType(FieldType::GetType())) - { - OutView = GetRangeView(); - return true; - } - return false; - } - -protected: - /** Construct a field range that contains exactly one field. */ - constexpr inline explicit TCbFieldIterator(FieldType InField) : FieldType(std::move(InField)), FieldsEnd(FieldType::GetPayloadEnd()) {} - - /** - * Construct a field range from the first field and a pointer to the end of the last field. - * - * @param InField The first field, or the default field if there are no fields. - * @param InFieldsEnd A pointer to the end of the payload of the last field, or null. - */ - constexpr inline TCbFieldIterator(FieldType&& InField, const void* InFieldsEnd) : FieldType(std::move(InField)), FieldsEnd(InFieldsEnd) - { - } - - /** Returns the end of the last field, or null for an iterator at the end. */ - template<typename OtherFieldType> - static inline const void* GetFieldsEnd(const TCbFieldIterator<OtherFieldType>& It) - { - return It.FieldsEnd; - } - -private: - friend inline TCbFieldIterator begin(const TCbFieldIterator& Iterator) { return Iterator; } - friend inline TCbFieldIterator end(const TCbFieldIterator&) { return TCbFieldIterator(); } - -private: - template<typename OtherType> - friend class TCbFieldIterator; - - friend class CbFieldViewIterator; - - friend class CbFieldIterator; - - /** Pointer to the first byte past the end of the last field. Set to null at the end. */ - const void* FieldsEnd = nullptr; -}; - -/** - * Iterator for CbField. - * - * @see CbFieldIterator - */ -class CbFieldViewIterator : public TCbFieldIterator<CbFieldView> -{ -public: - constexpr CbFieldViewIterator() = default; - - /** Construct a field range that contains exactly one field. */ - static inline CbFieldViewIterator MakeSingle(const CbFieldView& Field) { return CbFieldViewIterator(Field); } - - /** - * Construct a field range from a buffer containing zero or more valid fields. - * - * @param View A buffer containing zero or more valid fields. - * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. - */ - static inline CbFieldViewIterator MakeRange(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType) - { - return !View.IsEmpty() ? TCbFieldIterator(CbFieldView(View.GetData(), Type), View.GetDataEnd()) : CbFieldViewIterator(); - } - - /** Construct an iterator from another iterator. */ - template<typename OtherFieldType> - inline CbFieldViewIterator(const TCbFieldIterator<OtherFieldType>& It) - : TCbFieldIterator(ImplicitConv<CbFieldView>(It), GetFieldsEnd(It)) - { - } - -private: - using TCbFieldIterator::TCbFieldIterator; -}; - -/** - * Serialize a compact binary array to JSON. - */ -ZENCORE_API void CompactBinaryToJson(const CbArrayView& Object, StringBuilderBase& Builder); - -/** - * Array of CbField that have no names. - * - * Accessing a field of the array requires iteration. Access by index is not provided because the - * cost of accessing an item by index scales linearly with the index. - * - * This type only provides a view into memory and does not perform any memory management itself. - * Use CbArrayRef to hold a reference to the underlying memory when necessary. - */ -class CbArrayView : protected CbFieldView -{ - friend class CbFieldView; - -public: - /** @see CbField::CbField */ - using CbFieldView::CbFieldView; - - /** Construct an array with no fields. */ - ZENCORE_API CbArrayView(); - - /** Returns the number of items in the array. */ - ZENCORE_API uint64_t Num() const; - - /** Create an iterator for the fields of this array. */ - ZENCORE_API CbFieldViewIterator CreateViewIterator() const; - - /** Visit the fields of this array. */ - ZENCORE_API void VisitFields(ICbVisitor& Visitor); - - /** Access the array as an array field. */ - inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } - - /** Construct an array from an array field. No type check is performed! */ - static inline CbArrayView FromFieldView(const CbFieldView& Field) { return CbArrayView(Field); } - - /** Whether the array has any fields. */ - inline explicit operator bool() const { return Num() > 0; } - - /** Returns the size of the array in bytes if serialized by itself with no name. */ - ZENCORE_API uint64_t GetSize() const; - - /** Calculate the hash of the array if serialized by itself with no name. */ - ZENCORE_API IoHash GetHash() const; - - ZENCORE_API void GetHash(IoHashStream& Stream) const; - - /** - * Whether this array is identical to the other array. - * - * Performs a deep comparison of any contained arrays or objects and their fields. Comparison - * assumes that both fields are valid and are written in the canonical format. Fields must be - * written in the same order in arrays and objects, and name comparison is case sensitive. If - * these assumptions do not hold, this may return false for equivalent inputs. Validation can - * be done with the All mode to check these assumptions about the format of the inputs. - */ - ZENCORE_API bool Equals(const CbArrayView& Other) const; - - /** Copy the array into a buffer of exactly GetSize() bytes, with no name. */ - ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; - - /** Copy the array into an archive, including its type and name. */ - ZENCORE_API void CopyTo(BinaryWriter& Ar) const; - - ///** Invoke the visitor for every attachment in the array. */ - inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const - { - CreateViewIterator().IterateRangeAttachments(Visitor); - } - - /** Returns a view of the array, including the type and name when present. */ - using CbFieldView::GetView; - - StringBuilderBase& ToJson(StringBuilderBase& Builder) const - { - CompactBinaryToJson(*this, Builder); - return Builder; - } - -private: - friend inline CbFieldViewIterator begin(const CbArrayView& Array) { return Array.CreateViewIterator(); } - friend inline CbFieldViewIterator end(const CbArrayView&) { return CbFieldViewIterator(); } - - /** Construct an array from an array field. No type check is performed! Use via FromField. */ - inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {} -}; - -/** - * Serialize a compact binary object to JSON. - */ -ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder); - -class CbObjectView : protected CbFieldView -{ - friend class CbFieldView; - -public: - /** @see CbField::CbField */ - using CbFieldView::CbFieldView; - - using CbFieldView::TryGetSerializedView; - - /** Construct an object with no fields. */ - ZENCORE_API CbObjectView(); - - /** Create an iterator for the fields of this object. */ - ZENCORE_API CbFieldViewIterator CreateViewIterator() const; - - /** Visit the fields of this object. */ - ZENCORE_API void VisitFields(ICbVisitor& Visitor); - - /** - * Find a field by case-sensitive name comparison. - * - * The cost of this operation scales linearly with the number of fields in the object. Prefer - * to iterate over the fields only once when consuming an object. - * - * @param Name The name of the field. - * @return The matching field if found, otherwise a field with no value. - */ - ZENCORE_API CbFieldView FindView(std::string_view Name) const; - - /** Find a field by case-insensitive name comparison. */ - ZENCORE_API CbFieldView FindViewIgnoreCase(std::string_view Name) const; - - /** Find a field by case-sensitive name comparison. */ - inline CbFieldView operator[](std::string_view Name) const { return FindView(Name); } - - /** Access the object as an object field. */ - inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } - - /** Construct an object from an object field. No type check is performed! */ - static inline CbObjectView FromFieldView(const CbFieldView& Field) { return CbObjectView(Field); } - - /** Whether the object has any fields. */ - ZENCORE_API explicit operator bool() const; - - /** Returns the size of the object in bytes if serialized by itself with no name. */ - ZENCORE_API uint64_t GetSize() const; - - /** Calculate the hash of the object if serialized by itself with no name. */ - ZENCORE_API IoHash GetHash() const; - - ZENCORE_API void GetHash(IoHashStream& HashStream) const; - - /** - * Whether this object is identical to the other object. - * - * Performs a deep comparison of any contained arrays or objects and their fields. Comparison - * assumes that both fields are valid and are written in the canonical format. Fields must be - * written in the same order in arrays and objects, and name comparison is case sensitive. If - * these assumptions do not hold, this may return false for equivalent inputs. Validation can - * be done with the All mode to check these assumptions about the format of the inputs. - */ - ZENCORE_API bool Equals(const CbObjectView& Other) const; - - /** Copy the object into a buffer of exactly GetSize() bytes, with no name. */ - ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; - - /** Copy the field into an archive, including its type and name. */ - ZENCORE_API void CopyTo(BinaryWriter& Ar) const; - - ///** Invoke the visitor for every attachment in the object. */ - inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const - { - CreateViewIterator().IterateRangeAttachments(Visitor); - } - - /** Returns a view of the object, including the type and name when present. */ - using CbFieldView::GetView; - - /** Whether the field has a value. */ - using CbFieldView::operator bool; - - StringBuilderBase& ToJson(StringBuilderBase& Builder) const - { - CompactBinaryToJson(*this, Builder); - return Builder; - } - -private: - friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); } - friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); } - - /** Construct an object from an object field. No type check is performed! Use via FromField. */ - inline explicit CbObjectView(const CbFieldView& Field) : CbFieldView(Field) {} -}; - -////////////////////////////////////////////////////////////////////////// - -/** A reference to a function that is used to allocate buffers for compact binary data. */ -using BufferAllocator = std::function<UniqueBuffer(uint64_t Size)>; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** A wrapper that holds a reference to the buffer that contains its compact binary value. */ -template<typename BaseType> -class CbBuffer : public BaseType -{ -public: - /** Construct a default value. */ - CbBuffer() = default; - - /** - * Construct a value from a pointer to its data and an optional externally-provided type. - * - * @param ValueBuffer A buffer that exactly contains the value. - * @param Type HasFieldType means that ValueBuffer contains the type. Otherwise, use the given type. - */ - inline explicit CbBuffer(SharedBuffer ValueBuffer, CbFieldType Type = CbFieldType::HasFieldType) - { - if (ValueBuffer) - { - BaseType::operator=(BaseType(ValueBuffer.GetData(), Type)); - ZEN_ASSERT(ValueBuffer.GetView().Contains(BaseType::GetView())); - Buffer = std::move(ValueBuffer); - } - } - - /** Construct a value that holds a reference to the buffer that contains it. */ - inline CbBuffer(const BaseType& Value, SharedBuffer OuterBuffer) : BaseType(Value) - { - if (OuterBuffer) - { - ZEN_ASSERT(OuterBuffer.GetView().Contains(BaseType::GetView())); - Buffer = std::move(OuterBuffer); - } - } - - /** Construct a value that holds a reference to the buffer of the outer that contains it. */ - template<typename OtherBaseType> - inline CbBuffer(const BaseType& Value, CbBuffer<OtherBaseType> OuterRef) : CbBuffer(Value, std::move(OuterRef.Buffer)) - { - } - - /** Reset this to a default value and null buffer. */ - inline void Reset() { *this = CbBuffer(); } - - /** Whether this reference has ownership of the memory in its buffer. */ - inline bool IsOwned() const { return Buffer && Buffer.IsOwned(); } - - /** Clone the value, if necessary, to a buffer that this reference has ownership of. */ - inline void MakeOwned() - { - if (!IsOwned()) - { - UniqueBuffer MutableBuffer = UniqueBuffer::Alloc(BaseType::GetSize()); - BaseType::CopyTo(MutableBuffer); - BaseType::operator=(BaseType(MutableBuffer.GetData())); - Buffer = std::move(MutableBuffer); - } - } - - /** Returns a buffer that exactly contains this value. */ - inline SharedBuffer GetBuffer() const - { - const MemoryView View = BaseType::GetView(); - const SharedBuffer& OuterBuffer = GetOuterBuffer(); - return View == OuterBuffer.GetView() ? OuterBuffer : SharedBuffer::MakeView(View, OuterBuffer); - } - - /** Returns the outer buffer (if any) that contains this value. */ - inline const SharedBuffer& GetOuterBuffer() const& { return Buffer; } - inline SharedBuffer GetOuterBuffer() && { return std::move(Buffer); } - -private: - template<typename OtherType> - friend class CbBuffer; - - SharedBuffer Buffer; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * Factory functions for types derived from CbBuffer. - * - * This uses the curiously recurring template pattern to construct the correct type of reference. - * The derived type inherits from CbBufferRef and this type to expose the factory functions. - */ -template<typename RefType, typename BaseType> -class CbBufferFactory -{ -public: - /** Construct a value from an owned clone of its memory. */ - static inline RefType Clone(const void* const Data) { return Clone(BaseType(Data)); } - - /** Construct a value from an owned clone of its memory. */ - static inline RefType Clone(const BaseType& Value) - { - RefType Ref = MakeView(Value); - Ref.MakeOwned(); - return Ref; - } - - /** Construct a value from a read-only view of its memory and its optional outer buffer. */ - static inline RefType MakeView(const void* const Data, SharedBuffer OuterBuffer = SharedBuffer()) - { - return MakeView(BaseType(Data), std::move(OuterBuffer)); - } - - /** Construct a value from a read-only view of its memory and its optional outer buffer. */ - static inline RefType MakeView(const BaseType& Value, SharedBuffer OuterBuffer = SharedBuffer()) - { - return RefType(Value, std::move(OuterBuffer)); - } -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class CbArray; -class CbObject; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * A field that can hold a reference to the memory that contains it. - * - * @see CbBufferRef - */ -class CbField : public CbBuffer<CbFieldView>, public CbBufferFactory<CbField, CbFieldView> -{ -public: - using CbBuffer::CbBuffer; - - /** Access the field as an object. Defaults to an empty object on error. */ - inline CbObject AsObject() &; - inline CbObject AsObject() &&; - - /** Access the field as an array. Defaults to an empty array on error. */ - inline CbArray AsArray() &; - inline CbArray AsArray() &&; - - /** Access the field as binary. Returns the provided default on error. */ - inline SharedBuffer AsBinary(const SharedBuffer& Default = SharedBuffer()) &; - inline SharedBuffer AsBinary(const SharedBuffer& Default = SharedBuffer()) &&; -}; - -/** - * Iterator for CbFieldRef. - * - * @see CbFieldIterator - */ -class CbFieldIterator : public TCbFieldIterator<CbField> -{ -public: - /** Construct a field range from an owned clone of a range. */ - ZENCORE_API static CbFieldIterator CloneRange(const CbFieldViewIterator& It); - - /** Construct a field range from an owned clone of a range. */ - static inline CbFieldIterator CloneRange(const CbFieldIterator& It) { return CloneRange(CbFieldViewIterator(It)); } - - /** Construct a field range that contains exactly one field. */ - static inline CbFieldIterator MakeSingle(CbField Field) { return CbFieldIterator(std::move(Field)); } - - /** - * Construct a field range from a buffer containing zero or more valid fields. - * - * @param Buffer A buffer containing zero or more valid fields. - * @param Type HasFieldType means that Buffer contains the type. Otherwise, use the given type. - */ - static inline CbFieldIterator MakeRange(SharedBuffer Buffer, CbFieldType Type = CbFieldType::HasFieldType) - { - if (Buffer.GetSize()) - { - const void* const DataEnd = Buffer.GetView().GetDataEnd(); - return CbFieldIterator(CbField(std::move(Buffer), Type), DataEnd); - } - return CbFieldIterator(); - } - - /** Construct a field range from an iterator and its optional outer buffer. */ - static inline CbFieldIterator MakeRangeView(const CbFieldViewIterator& It, SharedBuffer OuterBuffer = SharedBuffer()) - { - return CbFieldIterator(CbField(It, std::move(OuterBuffer)), GetFieldsEnd(It)); - } - - /** Construct an empty field range. */ - constexpr CbFieldIterator() = default; - - /** Clone the range, if necessary, to a buffer that this reference has ownership of. */ - inline void MakeRangeOwned() - { - if (!IsOwned()) - { - *this = CloneRange(*this); - } - } - - /** Returns a buffer that exactly contains the field range. */ - ZENCORE_API SharedBuffer GetRangeBuffer() const; - -private: - using TCbFieldIterator::TCbFieldIterator; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * An array that can hold a reference to the memory that contains it. - * - * @see CbBuffer - */ -class CbArray : public CbBuffer<CbArrayView>, public CbBufferFactory<CbArray, CbArrayView> -{ -public: - using CbBuffer::CbBuffer; - - /** Create an iterator for the fields of this array. */ - inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); } - - /** Access the array as an array field. */ - inline CbField AsField() const& { return CbField(CbArrayView::AsFieldView(), *this); } - - /** Access the array as an array field. */ - inline CbField AsField() && { return CbField(CbArrayView::AsFieldView(), std::move(*this)); } - -private: - friend inline CbFieldIterator begin(const CbArray& Array) { return Array.CreateIterator(); } - friend inline CbFieldIterator end(const CbArray&) { return CbFieldIterator(); } -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * An object that can hold a reference to the memory that contains it. - * - * @see CbBuffer - */ -class CbObject : public CbBuffer<CbObjectView>, public CbBufferFactory<CbObject, CbObjectView> -{ -public: - using CbBuffer::CbBuffer; - - /** Create an iterator for the fields of this object. */ - inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); } - - /** Find a field by case-sensitive name comparison. */ - inline CbField Find(std::string_view Name) const - { - if (CbFieldView Field = FindView(Name)) - { - return CbField(Field, *this); - } - return CbField(); - } - - /** Find a field by case-insensitive name comparison. */ - inline CbField FindIgnoreCase(std::string_view Name) const - { - if (CbFieldView Field = FindViewIgnoreCase(Name)) - { - return CbField(Field, *this); - } - return CbField(); - } - - /** Find a field by case-sensitive name comparison. */ - inline CbFieldView operator[](std::string_view Name) const { return Find(Name); } - - /** Access the object as an object field. */ - inline CbField AsField() const& { return CbField(CbObjectView::AsFieldView(), *this); } - - /** Access the object as an object field. */ - inline CbField AsField() && { return CbField(CbObjectView::AsFieldView(), std::move(*this)); } - -private: - friend inline CbFieldIterator begin(const CbObject& Object) { return Object.CreateIterator(); } - friend inline CbFieldIterator end(const CbObject&) { return CbFieldIterator(); } -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -inline CbObject -CbField::AsObject() & -{ - return IsObject() ? CbObject(AsObjectView(), *this) : CbObject(); -} - -inline CbObject -CbField::AsObject() && -{ - return IsObject() ? CbObject(AsObjectView(), std::move(*this)) : CbObject(); -} - -inline CbArray -CbField::AsArray() & -{ - return IsArray() ? CbArray(AsArrayView(), *this) : CbArray(); -} - -inline CbArray -CbField::AsArray() && -{ - return IsArray() ? CbArray(AsArrayView(), std::move(*this)) : CbArray(); -} - -inline SharedBuffer -CbField::AsBinary(const SharedBuffer& Default) & -{ - const MemoryView View = AsBinaryView(); - return !HasError() ? SharedBuffer::MakeView(View, GetOuterBuffer()) : Default; -} - -inline SharedBuffer -CbField::AsBinary(const SharedBuffer& Default) && -{ - const MemoryView View = AsBinaryView(); - return !HasError() ? SharedBuffer::MakeView(View, std::move(*this).GetOuterBuffer()) : Default; -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * Load a compact binary field from an archive. - * - * The field may be an array or an object, which the caller can convert to by using AsArray or - * AsObject as appropriate. The buffer allocator is called to provide the buffer for the field - * to load into once its size has been determined. - * - * @param Ar Archive to read the field from. An error state is set on failure. - * @param Allocator Allocator for the buffer that the field is loaded into. - * @return A field with a reference to the allocated buffer, or a default field on failure. - */ -ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator); - -ZENCORE_API CbObject LoadCompactBinaryObject(IoBuffer&& Payload); -ZENCORE_API CbObject LoadCompactBinaryObject(const IoBuffer& Payload); -ZENCORE_API CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload); -ZENCORE_API CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload); - -/** - * Load a compact binary from JSON. - */ -ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json, std::string& Error); -ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json); - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * Determine the size in bytes of the compact binary field at the start of the view. - * - * This may be called on an incomplete or invalid field, in which case the returned size is zero. - * A size can always be extracted from a valid field with no name if a view of at least the first - * 10 bytes is provided, regardless of field size. For fields with names, the size of view needed - * to calculate a size is at most 10 + MaxNameLen + MeasureVarUInt(MaxNameLen). - * - * This function can be used when streaming a field, for example, to determine the size of buffer - * to fill before attempting to construct a field from it. - * - * @param View A memory view that may contain the start of a field. - * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. - */ -ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType); - -/** - * Try to determine the type and size of the compact binary field at the start of the view. - * - * This may be called on an incomplete or invalid field, in which case it will return false, with - * OutSize being 0 for invalid fields, otherwise the minimum view size necessary to make progress - * in measuring the field on the next call to this function. - * - * @note A return of true from this function does not indicate that the entire field is valid. - * - * @param InView A memory view that may contain the start of a field. - * @param OutType The type (with flags) of the field. None is written until a value is available. - * @param OutSize The total field size for a return of true, 0 for invalid fields, or the size to - * make progress in measuring the field on the next call to this function. - * @param InType HasFieldType means that InView contains the type. Otherwise, use the given type. - * @return true if the size of the field was determined, otherwise false. - */ -ZENCORE_API bool TryMeasureCompactBinary(MemoryView InView, - CbFieldType& OutType, - uint64_t& OutSize, - CbFieldType InType = CbFieldType::HasFieldType); - -inline CbFieldViewIterator -begin(CbFieldView& View) -{ - if (View.IsArray()) - { - return View.AsArrayView().CreateViewIterator(); - } - else if (View.IsObject()) - { - return View.AsObjectView().CreateViewIterator(); - } - - return CbFieldViewIterator(); -} - -inline CbFieldViewIterator -end(CbFieldView&) -{ - return CbFieldViewIterator(); -} - -void uson_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h deleted file mode 100644 index 4be8c2ba5..000000000 --- a/zencore/include/zencore/compactbinarybuilder.h +++ /dev/null @@ -1,661 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -#include <zencore/compactbinary.h> - -#include <zencore/enumflags.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/refcount.h> -#include <zencore/sha1.h> - -#include <atomic> -#include <memory> -#include <string> -#include <string_view> -#include <type_traits> -#include <vector> - -#include <gsl/gsl-lite.hpp> - -namespace zen { - -class CbAttachment; -class BinaryWriter; - -/** - * A writer for compact binary object, arrays, and fields. - * - * The writer produces a sequence of fields that can be saved to a provided memory buffer or into - * a new owned buffer. The typical use case is to write a single object, which can be accessed by - * calling Save().AsObjectRef() or Save(Buffer).AsObject(). - * - * The writer will assert on most incorrect usage and will always produce valid compact binary if - * provided with valid input. The writer does not check for invalid UTF-8 string encoding, object - * fields with duplicate names, or invalid compact binary being copied from another source. - * - * It is most convenient to use the streaming API for the writer, as demonstrated in the example. - * - * When writing a small amount of compact binary data, TCbWriter can be more efficient as it uses - * a fixed-size stack buffer for storage before spilling onto the heap. - * - * @see TCbWriter - * - * Example: - * - * CbObjectRef WriteObject() - * { - * CbWriter<256> Writer; - * Writer.BeginObject(); - * - * Writer << "Resize" << true; - * Writer << "MaxWidth" << 1024; - * Writer << "MaxHeight" << 1024; - * - * Writer.BeginArray(); - * Writer << "FormatA" << "FormatB" << "FormatC"; - * Writer.EndArray(); - * - * Writer.EndObject(); - * return Writer.Save().AsObjectRef(); - * } - */ -class CbWriter -{ -public: - ZENCORE_API CbWriter(); - ZENCORE_API ~CbWriter(); - - CbWriter(const CbWriter&) = delete; - CbWriter& operator=(const CbWriter&) = delete; - - /** Empty the writer without releasing any allocated memory. */ - ZENCORE_API void Reset(); - - /** - * Serialize the field(s) to an owned buffer and return it as an iterator. - * - * It is not valid to call this function in the middle of writing an object, array, or field. - * The writer remains valid for further use when this function returns. - */ - ZENCORE_API CbFieldIterator Save(); - - /** - * Serialize the field(s) to memory. - * - * It is not valid to call this function in the middle of writing an object, array, or field. - * The writer remains valid for further use when this function returns. - * - * @param Buffer A mutable memory view to write to. Must be exactly GetSaveSize() bytes. - * @return An iterator for the field(s) written to the buffer. - */ - ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer); - - ZENCORE_API void Save(BinaryWriter& Writer); - - /** - * The size of buffer (in bytes) required to serialize the fields that have been written. - * - * It is not valid to call this function in the middle of writing an object, array, or field. - */ - ZENCORE_API uint64_t GetSaveSize() const; - - /** - * Sets the name of the next field to be written. - * - * It is not valid to call this function when writing a field inside an array. - * Names must be valid UTF-8 and must be unique within an object. - */ - ZENCORE_API CbWriter& SetName(std::string_view Name); - - /** Copy the value (not the name) of an existing field. */ - inline void AddField(std::string_view Name, const CbFieldView& Value) - { - SetName(Name); - AddField(Value); - } - - ZENCORE_API void AddField(const CbFieldView& Value); - - /** Copy the value (not the name) of an existing field. Holds a reference if owned. */ - inline void AddField(std::string_view Name, const CbField& Value) - { - SetName(Name); - AddField(Value); - } - ZENCORE_API void AddField(const CbField& Value); - - /** Begin a new object. Must have a matching call to EndObject. */ - inline void BeginObject(std::string_view Name) - { - SetName(Name); - BeginObject(); - } - ZENCORE_API void BeginObject(); - /** End an object after its fields have been written. */ - ZENCORE_API void EndObject(); - - /** Copy the value (not the name) of an existing object. */ - inline void AddObject(std::string_view Name, const CbObjectView& Value) - { - SetName(Name); - AddObject(Value); - } - ZENCORE_API void AddObject(const CbObjectView& Value); - /** Copy the value (not the name) of an existing object. Holds a reference if owned. */ - inline void AddObject(std::string_view Name, const CbObject& Value) - { - SetName(Name); - AddObject(Value); - } - ZENCORE_API void AddObject(const CbObject& Value); - - /** Begin a new array. Must have a matching call to EndArray. */ - inline void BeginArray(std::string_view Name) - { - SetName(Name); - BeginArray(); - } - ZENCORE_API void BeginArray(); - /** End an array after its fields have been written. */ - ZENCORE_API void EndArray(); - - /** Copy the value (not the name) of an existing array. */ - inline void AddArray(std::string_view Name, const CbArrayView& Value) - { - SetName(Name); - AddArray(Value); - } - ZENCORE_API void AddArray(const CbArrayView& Value); - /** Copy the value (not the name) of an existing array. Holds a reference if owned. */ - inline void AddArray(std::string_view Name, const CbArray& Value) - { - SetName(Name); - AddArray(Value); - } - ZENCORE_API void AddArray(const CbArray& Value); - - /** Write a null field. */ - inline void AddNull(std::string_view Name) - { - SetName(Name); - AddNull(); - } - ZENCORE_API void AddNull(); - - /** Write a binary field by copying Size bytes from Value. */ - inline void AddBinary(std::string_view Name, const void* Value, uint64_t Size) - { - SetName(Name); - AddBinary(Value, Size); - } - ZENCORE_API void AddBinary(const void* Value, uint64_t Size); - /** Write a binary field by copying the view. */ - inline void AddBinary(std::string_view Name, MemoryView Value) - { - SetName(Name); - AddBinary(Value); - } - inline void AddBinary(MemoryView Value) { AddBinary(Value.GetData(), Value.GetSize()); } - - /** Write a binary field by copying the buffer. Holds a reference if owned. */ - inline void AddBinary(std::string_view Name, IoBuffer Value) - { - SetName(Name); - AddBinary(std::move(Value)); - } - ZENCORE_API void AddBinary(IoBuffer Value); - ZENCORE_API void AddBinary(SharedBuffer Value); - - inline void AddBinary(std::string_view Name, const CompositeBuffer& Buffer) - { - SetName(Name); - AddBinary(Buffer); - } - ZENCORE_API void AddBinary(const CompositeBuffer& Buffer); - - /** Write a string field by copying the UTF-8 value. */ - inline void AddString(std::string_view Name, std::string_view Value) - { - SetName(Name); - AddString(Value); - } - ZENCORE_API void AddString(std::string_view Value); - /** Write a string field by converting the UTF-16 value to UTF-8. */ - inline void AddString(std::string_view Name, std::wstring_view Value) - { - SetName(Name); - AddString(Value); - } - ZENCORE_API void AddString(std::wstring_view Value); - - /** Write an integer field. */ - inline void AddInteger(std::string_view Name, int32_t Value) - { - SetName(Name); - AddInteger(Value); - } - ZENCORE_API void AddInteger(int32_t Value); - /** Write an integer field. */ - inline void AddInteger(std::string_view Name, int64_t Value) - { - SetName(Name); - AddInteger(Value); - } - ZENCORE_API void AddInteger(int64_t Value); - /** Write an integer field. */ - inline void AddInteger(std::string_view Name, uint32_t Value) - { - SetName(Name); - AddInteger(Value); - } - ZENCORE_API void AddInteger(uint32_t Value); - /** Write an integer field. */ - inline void AddInteger(std::string_view Name, uint64_t Value) - { - SetName(Name); - AddInteger(Value); - } - ZENCORE_API void AddInteger(uint64_t Value); - - /** Write a float field from a 32-bit float value. */ - inline void AddFloat(std::string_view Name, float Value) - { - SetName(Name); - AddFloat(Value); - } - ZENCORE_API void AddFloat(float Value); - - /** Write a float field from a 64-bit float value. */ - inline void AddFloat(std::string_view Name, double Value) - { - SetName(Name); - AddFloat(Value); - } - ZENCORE_API void AddFloat(double Value); - - /** Write a bool field. */ - inline void AddBool(std::string_view Name, bool bValue) - { - SetName(Name); - AddBool(bValue); - } - ZENCORE_API void AddBool(bool bValue); - - /** Write a field referencing a compact binary attachment by its hash. */ - inline void AddObjectAttachment(std::string_view Name, const IoHash& Value) - { - SetName(Name); - AddObjectAttachment(Value); - } - ZENCORE_API void AddObjectAttachment(const IoHash& Value); - - /** Write a field referencing a binary attachment by its hash. */ - inline void AddBinaryAttachment(std::string_view Name, const IoHash& Value) - { - SetName(Name); - AddBinaryAttachment(Value); - } - ZENCORE_API void AddBinaryAttachment(const IoHash& Value); - - /** Write a field referencing the attachment by its hash. */ - inline void AddAttachment(std::string_view Name, const CbAttachment& Attachment) - { - SetName(Name); - AddAttachment(Attachment); - } - ZENCORE_API void AddAttachment(const CbAttachment& Attachment); - - /** Write a hash field. */ - inline void AddHash(std::string_view Name, const IoHash& Value) - { - SetName(Name); - AddHash(Value); - } - ZENCORE_API void AddHash(const IoHash& Value); - - /** Write a UUID field. */ - inline void AddUuid(std::string_view Name, const Guid& Value) - { - SetName(Name); - AddUuid(Value); - } - ZENCORE_API void AddUuid(const Guid& Value); - - /** Write an ObjectId field. */ - inline void AddObjectId(std::string_view Name, const Oid& Value) - { - SetName(Name); - AddObjectId(Value); - } - ZENCORE_API void AddObjectId(const Oid& Value); - - /** Write a date/time field with the specified count of 100ns ticks since the epoch. */ - inline void AddDateTimeTicks(std::string_view Name, int64_t Ticks) - { - SetName(Name); - AddDateTimeTicks(Ticks); - } - ZENCORE_API void AddDateTimeTicks(int64_t Ticks); - - /** Write a date/time field. */ - inline void AddDateTime(std::string_view Name, DateTime Value) - { - SetName(Name); - AddDateTime(Value); - } - ZENCORE_API void AddDateTime(DateTime Value); - - /** Write a time span field with the specified count of 100ns ticks. */ - inline void AddTimeSpanTicks(std::string_view Name, int64_t Ticks) - { - SetName(Name); - AddTimeSpanTicks(Ticks); - } - ZENCORE_API void AddTimeSpanTicks(int64_t Ticks); - - /** Write a time span field. */ - inline void AddTimeSpan(std::string_view Name, TimeSpan Value) - { - SetName(Name); - AddTimeSpan(Value); - } - ZENCORE_API void AddTimeSpan(TimeSpan Value); - - /** Private flags that are public to work with ENUM_CLASS_FLAGS. */ - enum class StateFlags : uint8_t; - -protected: - /** Reserve the specified size up front until the format is optimized. */ - ZENCORE_API explicit CbWriter(int64_t InitialSize); - -private: - friend CbWriter& operator<<(CbWriter& Writer, std::string_view NameOrValue); - - /** Begin writing a field. May be called twice for named fields. */ - void BeginField(); - - /** Finish writing a field by writing its type. */ - void EndField(CbFieldType Type); - - /** Set the field name if valid in this state, otherwise write add a string field. */ - ZENCORE_API void SetNameOrAddString(std::string_view NameOrValue); - - /** Returns a view of the name of the active field, if any, otherwise the empty view. */ - std::string_view GetActiveName() const; - - /** Remove field types after the first to make the sequence uniform. */ - void MakeFieldsUniform(int64_t FieldBeginOffset, int64_t FieldEndOffset); - - /** State of the object, array, or top-level field being written. */ - struct WriterState - { - StateFlags Flags{}; - /** The type of the fields in the sequence if uniform, otherwise None. */ - CbFieldType UniformType{}; - /** The offset of the start of the current field. */ - int64_t Offset{}; - /** The number of fields written in this state. */ - uint64_t Count{}; - }; - -private: - // This is a prototype-quality format for the writer. Using an array of bytes is inefficient, - // and will lead to many unnecessary copies and moves of the data to resize the array, insert - // object and array sizes, and remove field types for uniform objects and uniform arrays. The - // optimized format will be a list of power-of-two blocks and an optional first block that is - // provided externally, such as on the stack. That format will store the offsets that require - // object or array sizes to be inserted and field types to be removed, and will perform those - // operations only when saving to a buffer. - std::vector<uint8_t> Data; - std::vector<WriterState> States; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * A writer for compact binary object, arrays, and fields that uses a fixed-size stack buffer. - * - * @see CbWriter - */ -template<uint32_t InlineBufferSize> -class FixedCbWriter : public CbWriter -{ -public: - inline FixedCbWriter() : CbWriter(InlineBufferSize) {} - - FixedCbWriter(const FixedCbWriter&) = delete; - FixedCbWriter& operator=(const FixedCbWriter&) = delete; - -private: - // Reserve the inline buffer now even though we are unable to use it. This will avoid causing - // new stack overflows when this functionality is properly implemented in the future. - uint8_t Buffer[InlineBufferSize]; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -class CbObjectWriter : public CbWriter -{ -public: - CbObjectWriter() { BeginObject(); } - - ZENCORE_API CbObject Save() - { - Finalize(); - return CbWriter::Save().AsObject(); - } - - ZENCORE_API void Save(BinaryWriter& Writer) - { - Finalize(); - return CbWriter::Save(Writer); - } - - ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer) - { - ZEN_ASSERT(m_Finalized); - return CbWriter::Save(Buffer); - } - - uint64_t GetSaveSize() - { - ZEN_ASSERT(m_Finalized); - return CbWriter::GetSaveSize(); - } - - void Finalize() - { - if (m_Finalized == false) - { - EndObject(); - m_Finalized = true; - } - } - - CbObjectWriter(const CbWriter&) = delete; - CbObjectWriter& operator=(const CbWriter&) = delete; - -private: - bool m_Finalized = false; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** Write the field name if valid in this state, otherwise write the string value. */ -inline CbWriter& -operator<<(CbWriter& Writer, std::string_view NameOrValue) -{ - Writer.SetNameOrAddString(NameOrValue); - return Writer; -} - -/** Write the field name if valid in this state, otherwise write the string value. */ -inline CbWriter& -operator<<(CbWriter& Writer, const char* NameOrValue) -{ - return Writer << std::string_view(NameOrValue); -} - -inline CbWriter& -operator<<(CbWriter& Writer, const CbFieldView& Value) -{ - Writer.AddField(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const CbField& Value) -{ - Writer.AddField(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const CbObjectView& Value) -{ - Writer.AddObject(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const CbObject& Value) -{ - Writer.AddObject(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const CbArrayView& Value) -{ - Writer.AddArray(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const CbArray& Value) -{ - Writer.AddArray(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, std::nullptr_t) -{ - Writer.AddNull(); - return Writer; -} - -#if defined(__clang__) && defined(__APPLE__) -/* Apple Clang has different types for uint64_t and size_t so an override is - needed here. Without it, Clang can't disambiguate integer overloads */ -inline CbWriter& -operator<<(CbWriter& Writer, std::size_t Value) -{ - Writer.AddInteger(uint64_t(Value)); - return Writer; -} -#endif - -inline CbWriter& -operator<<(CbWriter& Writer, std::wstring_view Value) -{ - Writer.AddString(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const wchar_t* Value) -{ - Writer.AddString(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, int32_t Value) -{ - Writer.AddInteger(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, int64_t Value) -{ - Writer.AddInteger(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, uint32_t Value) -{ - Writer.AddInteger(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, uint64_t Value) -{ - Writer.AddInteger(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, float Value) -{ - Writer.AddFloat(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, double Value) -{ - Writer.AddFloat(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, bool Value) -{ - Writer.AddBool(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const CbAttachment& Attachment) -{ - Writer.AddAttachment(Attachment); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const IoHash& Value) -{ - Writer.AddHash(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const Guid& Value) -{ - Writer.AddUuid(Value); - return Writer; -} - -inline CbWriter& -operator<<(CbWriter& Writer, const Oid& Value) -{ - Writer.AddObjectId(Value); - return Writer; -} - -ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value); -ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value); - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -void usonbuilder_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/compactbinarypackage.h b/zencore/include/zencore/compactbinarypackage.h deleted file mode 100644 index 16f723edc..000000000 --- a/zencore/include/zencore/compactbinarypackage.h +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -#include <zencore/compactbinary.h> -#include <zencore/compress.h> -#include <zencore/iohash.h> - -#include <functional> -#include <span> -#include <variant> - -#ifdef GetObject -# error "windows.h pollution" -# undef GetObject -#endif - -namespace zen { - -class CbWriter; -class BinaryReader; -class BinaryWriter; -class IoBuffer; -class CbAttachment; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * An attachment is either binary or compact binary and is identified by its hash. - * - * A compact binary attachment is also a valid binary attachment and may be accessed as binary. - * - * Attachments are serialized as one or two compact binary fields with no name. A Binary field is - * written first with its content. The content hash is omitted when the content size is zero, and - * is otherwise written as a BinaryReference or CompactBinaryReference depending on the type. - */ -class CbAttachment -{ -public: - /** Construct a null attachment. */ - CbAttachment() = default; - - /** Construct a compact binary attachment. Value is cloned if not owned. */ - inline explicit CbAttachment(const CbObject& InValue) : CbAttachment(InValue, nullptr) {} - - /** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */ - inline explicit CbAttachment(const CbObject& InValue, const IoHash& Hash) : CbAttachment(InValue, &Hash) {} - - /** Construct a raw binary attachment. Value is cloned if not owned. */ - ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue); - - /** Construct a raw binary attachment. Value is cloned if not owned. Hash must match Value. */ - ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue, const IoHash& Hash); - - /** Construct a raw binary attachment. Value is cloned if not owned. */ - ZENCORE_API explicit CbAttachment(const CompositeBuffer& InValue); - - /** Construct a raw binary attachment. Value is cloned if not owned. */ - ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue); - - /** Construct a raw binary attachment. Value is cloned if not owned. */ - ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash); - - /** Construct a compressed binary attachment. Value is cloned if not owned. */ - ZENCORE_API explicit CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash); - ZENCORE_API explicit CbAttachment(CompressedBuffer&& InValue, const IoHash& Hash); - - /** Reset this to a null attachment. */ - inline void Reset() { *this = CbAttachment(); } - - /** Whether the attachment has a value. */ - inline explicit operator bool() const { return !IsNull(); } - - /** Whether the attachment has a value. */ - ZENCORE_API [[nodiscard]] bool IsNull() const; - - /** Access the attachment as binary. Defaults to a null buffer on error. */ - ZENCORE_API [[nodiscard]] SharedBuffer AsBinary() const; - - /** Access the attachment as raw binary. Defaults to a null buffer on error. */ - ZENCORE_API [[nodiscard]] CompositeBuffer AsCompositeBinary() const; - - /** Access the attachment as compressed binary. Defaults to a null buffer if the attachment is null. */ - ZENCORE_API [[nodiscard]] CompressedBuffer AsCompressedBinary() const; - - /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ - ZENCORE_API [[nodiscard]] CbObject AsObject() const; - - /** Returns true if the attachment is binary */ - ZENCORE_API [[nodiscard]] bool IsBinary() const; - - /** Returns true if the attachment is compressed binary */ - ZENCORE_API [[nodiscard]] bool IsCompressedBinary() const; - - /** Returns whether the attachment is an object. */ - ZENCORE_API [[nodiscard]] bool IsObject() const; - - /** Returns the hash of the attachment value. */ - ZENCORE_API [[nodiscard]] IoHash GetHash() const; - - /** Compares attachments by their hash. Any discrepancy in type must be handled externally. */ - inline bool operator==(const CbAttachment& Attachment) const { return GetHash() == Attachment.GetHash(); } - inline bool operator!=(const CbAttachment& Attachment) const { return GetHash() != Attachment.GetHash(); } - inline bool operator<(const CbAttachment& Attachment) const { return GetHash() < Attachment.GetHash(); } - - /** - * Load the attachment from compact binary as written by Save. - * - * The attachment references the input iterator if it is owned, and otherwise clones the value. - * - * The iterator is advanced as attachment fields are consumed from it. - */ - ZENCORE_API bool TryLoad(CbFieldIterator& Fields); - - /** - * Load the attachment from compact binary as written by Save. - */ - ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc); - - /** - * Load the attachment from compact binary as written by Save. - */ - ZENCORE_API bool TryLoad(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc); - - /** Save the attachment into the writer as a stream of compact binary fields. */ - ZENCORE_API void Save(CbWriter& Writer) const; - - /** Save the attachment into the writer as a stream of compact binary fields. */ - ZENCORE_API void Save(BinaryWriter& Writer) const; - -private: - ZENCORE_API CbAttachment(const CbObject& Value, const IoHash* Hash); - - IoHash Hash; - std::variant<std::nullptr_t, CbObject, CompositeBuffer, CompressedBuffer> Value; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * A package is a compact binary object with attachments for its external references. - * - * A package is basically a Merkle tree with compact binary as its root and other non-leaf nodes, - * and either binary or compact binary as its leaf nodes. A node references its child nodes using - * BinaryHash or FieldHash fields in its compact binary representation. - * - * It is invalid for a package to include attachments that are not referenced by its object or by - * one of its referenced compact binary attachments. When attachments are added explicitly, it is - * the responsibility of the package creator to follow this requirement. Attachments that are not - * referenced may not survive a round-trip through certain storage systems. - * - * It is valid for a package to exclude referenced attachments, but then it is the responsibility - * of the package consumer to have a mechanism for resolving those references when necessary. - * - * A package is serialized as a sequence of compact binary fields with no name. The object may be - * both preceded and followed by attachments. The object itself is written as an Object field and - * followed by its hash in a CompactBinaryReference field when the object is non-empty. A package - * ends with a Null field. The canonical order of components is the object and its hash, followed - * by the attachments ordered by hash, followed by a Null field. It is valid for the a package to - * have its components serialized in any order, provided there is at most one object and the null - * field is written last. - */ -class CbPackage -{ -public: - /** - * A function that resolves a hash to a buffer containing the data matching that hash. - * - * The resolver may return a null buffer to skip resolving an attachment for the hash. - */ - using AttachmentResolver = std::function<SharedBuffer(const IoHash& Hash)>; - - /** Construct a null package. */ - CbPackage() = default; - - /** - * Construct a package from a root object without gathering attachments. - * - * @param InObject The root object, which will be cloned unless it is owned. - */ - inline explicit CbPackage(CbObject InObject) { SetObject(std::move(InObject)); } - - /** - * Construct a package from a root object and gather attachments using the resolver. - * - * @param InObject The root object, which will be cloned unless it is owned. - * @param InResolver A function that is invoked for every reference and binary reference field. - */ - inline explicit CbPackage(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), InResolver); } - - /** - * Construct a package from a root object without gathering attachments. - * - * @param InObject The root object, which will be cloned unless it is owned. - * @param InObjectHash The hash of the object, which must match to avoid validation errors. - */ - inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), InObjectHash); } - - /** - * Construct a package from a root object and gather attachments using the resolver. - * - * @param InObject The root object, which will be cloned unless it is owned. - * @param InObjectHash The hash of the object, which must match to avoid validation errors. - * @param InResolver A function that is invoked for every reference and binary reference field. - */ - inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver) - { - SetObject(std::move(InObject), InObjectHash, InResolver); - } - - /** Reset this to a null package. */ - inline void Reset() { *this = CbPackage(); } - - /** Whether the package has a non-empty object or attachments. */ - inline explicit operator bool() const { return !IsNull(); } - - /** Whether the package has an empty object and no attachments. */ - inline bool IsNull() const { return !Object && Attachments.size() == 0; } - - /** Returns the compact binary object for the package. */ - inline const CbObject& GetObject() const { return Object; } - - /** Returns the has of the compact binary object for the package. */ - inline const IoHash& GetObjectHash() const { return ObjectHash; } - - /** - * Set the root object without gathering attachments. - * - * @param InObject The root object, which will be cloned unless it is owned. - */ - inline void SetObject(CbObject InObject) { SetObject(std::move(InObject), nullptr, nullptr); } - - /** - * Set the root object and gather attachments using the resolver. - * - * @param InObject The root object, which will be cloned unless it is owned. - * @param InResolver A function that is invoked for every reference and binary reference field. - */ - inline void SetObject(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), nullptr, &InResolver); } - - /** - * Set the root object without gathering attachments. - * - * @param InObject The root object, which will be cloned unless it is owned. - * @param InObjectHash The hash of the object, which must match to avoid validation errors. - */ - inline void SetObject(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), &InObjectHash, nullptr); } - - /** - * Set the root object and gather attachments using the resolver. - * - * @param InObject The root object, which will be cloned unless it is owned. - * @param InObjectHash The hash of the object, which must match to avoid validation errors. - * @param InResolver A function that is invoked for every reference and binary reference field. - */ - inline void SetObject(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver) - { - SetObject(std::move(InObject), &InObjectHash, &InResolver); - } - - /** Returns the attachments in this package. */ - inline std::span<const CbAttachment> GetAttachments() const { return Attachments; } - - /** - * Find an attachment by its hash. - * - * @return The attachment, or null if the attachment is not found. - * @note The returned pointer is only valid until the attachments on this package are modified. - */ - ZENCORE_API const CbAttachment* FindAttachment(const IoHash& Hash) const; - - /** Find an attachment if it exists in the package. */ - inline const CbAttachment* FindAttachment(const CbAttachment& Attachment) const { return FindAttachment(Attachment.GetHash()); } - - /** Add the attachment to this package. */ - inline void AddAttachment(const CbAttachment& Attachment) { AddAttachment(Attachment, nullptr); } - - /** Add the attachment to this package, along with any references that can be resolved. */ - inline void AddAttachment(const CbAttachment& Attachment, AttachmentResolver Resolver) { AddAttachment(Attachment, &Resolver); } - - void AddAttachments(std::span<const CbAttachment> Attachments); - - /** - * Remove an attachment by hash. - * - * @return Number of attachments removed, which will be either 0 or 1. - */ - ZENCORE_API int32_t RemoveAttachment(const IoHash& Hash); - inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); } - - /** Compares packages by their object and attachment hashes. */ - ZENCORE_API bool Equals(const CbPackage& Package) const; - inline bool operator==(const CbPackage& Package) const { return Equals(Package); } - inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); } - - /** - * Load the object and attachments from compact binary as written by Save. - * - * The object and attachments reference the input iterator, if it is owned, and otherwise clones - * the object and attachments individually to make owned copies. - * - * The iterator is advanced as object and attachment fields are consumed from it. - */ - ZENCORE_API bool TryLoad(CbFieldIterator& Fields); - ZENCORE_API bool TryLoad(IoBuffer Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); - ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); - - /** Save the object and attachments into the writer as a stream of compact binary fields. */ - ZENCORE_API void Save(CbWriter& Writer) const; - - /** Save the object and attachments into the writer as a stream of compact binary fields. */ - ZENCORE_API void Save(BinaryWriter& Writer) const; - -private: - ZENCORE_API void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver); - ZENCORE_API void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver); - - void GatherAttachments(const CbObject& Object, AttachmentResolver Resolver); - - /** Attachments ordered by their hash. */ - std::vector<CbAttachment> Attachments; - CbObject Object; - IoHash ObjectHash; -}; - -namespace legacy { - void SaveCbAttachment(const CbAttachment& Attachment, CbWriter& Writer); - void SaveCbPackage(const CbPackage& Package, CbWriter& Writer); - void SaveCbPackage(const CbPackage& Package, BinaryWriter& Ar); - bool TryLoadCbPackage(CbPackage& Package, IoBuffer Buffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper = nullptr); - bool TryLoadCbPackage(CbPackage& Package, - BinaryReader& Reader, - BufferAllocator Allocator, - CbPackage::AttachmentResolver* Mapper = nullptr); -} // namespace legacy - -void usonpackage_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/compactbinaryvalidation.h b/zencore/include/zencore/compactbinaryvalidation.h deleted file mode 100644 index b1fab9572..000000000 --- a/zencore/include/zencore/compactbinaryvalidation.h +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -#include <zencore/compactbinary.h> -#include <zencore/enumflags.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/refcount.h> -#include <zencore/sha1.h> - -#include <gsl/gsl-lite.hpp> - -namespace zen { - -/** Flags for validating compact binary data. */ -enum class CbValidateMode : uint32_t -{ - /** Skip validation if no other validation modes are enabled. */ - None = 0, - - /** - * Validate that the value can be read and stays inside the bounds of the memory view. - * - * This is the minimum level of validation required to be able to safely read a field, array, - * or object without the risk of crashing or reading out of bounds. - */ - Default = 1 << 0, - - /** - * Validate that object fields have unique non-empty names and array fields have no names. - * - * Name validation failures typically do not inhibit reading the input, but duplicated fields - * cannot be looked up by name other than the first, and converting to other data formats can - * fail in the presence of naming issues. - */ - Names = 1 << 1, - - /** - * Validate that fields are serialized in the canonical format. - * - * Format validation failures typically do not inhibit reading the input. Values that fail in - * this mode require more memory than in the canonical format, and comparisons of such values - * for equality are not reliable. Examples of failures include uniform arrays or objects that - * were not encoded uniformly, variable-length integers that could be encoded in fewer bytes, - * or 64-bit floats that could be encoded in 32 bits without loss of precision. - */ - Format = 1 << 2, - - /** - * Validate that there is no padding after the value before the end of the memory view. - * - * Padding validation failures have no impact on the ability to read the input, but are using - * more memory than necessary. - */ - Padding = 1 << 3, - - /** - * Validate that a package or attachment has the expected fields. - */ - Package = 1 << 4, - - /** - * Validate that a package or attachment matches its saved hashes. - */ - PackageHash = 1 << 5, - - /** Perform all validation described above. */ - All = Default | Names | Format | Padding | Package | PackageHash, -}; - -ENUM_CLASS_FLAGS(CbValidateMode); - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** Flags for compact binary validation errors. Multiple flags may be combined. */ -enum class CbValidateError : uint32_t -{ - /** The input had no validation errors. */ - None = 0, - - // Mode: Default - - /** The input cannot be read without reading out of bounds. */ - OutOfBounds = 1 << 0, - /** The input has a field with an unrecognized or invalid type. */ - InvalidType = 1 << 1, - - // Mode: Names - - /** An object had more than one field with the same name. */ - DuplicateName = 1 << 2, - /** An object had a field with no name. */ - MissingName = 1 << 3, - /** An array field had a name. */ - ArrayName = 1 << 4, - - // Mode: Format - - /** A name or string payload is not valid UTF-8. */ - InvalidString = 1 << 5, - /** A size or integer payload can be encoded in fewer bytes. */ - InvalidInteger = 1 << 6, - /** A float64 payload can be encoded as a float32 without loss of precision. */ - InvalidFloat = 1 << 7, - /** An object has the same type for every field but is not uniform. */ - NonUniformObject = 1 << 8, - /** An array has the same type for every field and non-empty payloads but is not uniform. */ - NonUniformArray = 1 << 9, - - // Mode: Padding - - /** A value did not use the entire memory view given for validation. */ - Padding = 1 << 10, - - // Mode: Package - - /** The package or attachment had missing fields or fields out of order. */ - InvalidPackageFormat = 1 << 11, - /** The object or an attachment did not match the hash stored for it. */ - InvalidPackageHash = 1 << 12, - /** The package contained more than one copy of the same attachment. */ - DuplicateAttachments = 1 << 13, - /** The package contained more than one object. */ - MultiplePackageObjects = 1 << 14, - /** The package contained an object with no fields. */ - NullPackageObject = 1 << 15, - /** The package contained a null attachment. */ - NullPackageAttachment = 1 << 16, -}; - -ENUM_CLASS_FLAGS(CbValidateError); - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * Validate the compact binary data for one field in the view as specified by the mode flags. - * - * Only one top-level field is processed from the view, and validation recurses into any array or - * object within that field. To validate multiple consecutive top-level fields, call the function - * once for each top-level field. If the given view might contain multiple top-level fields, then - * either exclude the Padding flag from the Mode or use MeasureCompactBinary to break up the view - * into its constituent fields before validating. - * - * @param View A memory view containing at least one top-level field. - * @param Mode A combination of the flags for the types of validation to perform. - * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. - * @return None on success, otherwise the flags for the types of errors that were detected. - */ -ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType); - -/** - * Validate the compact binary data for every field in the view as specified by the mode flags. - * - * This function expects the entire view to contain fields. Any trailing region of the view which - * does not contain a valid field will produce an OutOfBounds or InvalidType error instead of the - * Padding error that would be produced by the single field validation function. - * - * @see ValidateCompactBinary - */ -ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode); - -/** - * Validate the compact binary attachment pointed to by the view as specified by the mode flags. - * - * The attachment is validated with ValidateCompactBinary by using the validation mode specified. - * Include ECbValidateMode::Package to validate the attachment format and hash. - * - * @see ValidateCompactBinary - * - * @param View A memory view containing a package. - * @param Mode A combination of the flags for the types of validation to perform. - * @return None on success, otherwise the flags for the types of errors that were detected. - */ -ZENCORE_API CbValidateError ValidateObjectAttachment(MemoryView View, CbValidateMode Mode); - -/** - * Validate the compact binary package pointed to by the view as specified by the mode flags. - * - * The package, and attachments, are validated with ValidateCompactBinary by using the validation - * mode specified. Include ECbValidateMode::Package to validate the package format and hashes. - * - * @see ValidateCompactBinary - * - * @param View A memory view containing a package. - * @param Mode A combination of the flags for the types of validation to perform. - * @return None on success, otherwise the flags for the types of errors that were detected. - */ -ZENCORE_API CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode); - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -void usonvalidation_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/compactbinaryvalue.h b/zencore/include/zencore/compactbinaryvalue.h deleted file mode 100644 index 0124a8983..000000000 --- a/zencore/include/zencore/compactbinaryvalue.h +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/compactbinary.h> -#include <zencore/endian.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/memory.h> - -namespace zen { - -namespace CompactBinaryPrivate { - - template<typename T> - static constexpr inline T ReadUnaligned(const void* const Memory) - { -#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS - return *static_cast<const T*>(Memory); -#else - T Value; - memcpy(&Value, Memory, sizeof(Value)); - return Value; -#endif - } -} // namespace CompactBinaryPrivate -/** - * A type that provides unchecked access to compact binary values. - * - * The main purpose of the type is to efficiently switch on field type. For every other use case, - * prefer to use the field, array, and object types directly. The accessors here do not check the - * type before reading the value, which means they can read out of bounds even on a valid compact - * binary value if the wrong accessor is used. - */ -class CbValue -{ -public: - CbValue(CbFieldType Type, const void* Value); - - CbObjectView AsObjectView() const; - CbArrayView AsArrayView() const; - - MemoryView AsBinary() const; - - /** Access as a string. Checks for range errors and uses the default if OutError is not null. */ - std::string_view AsString(CbFieldError* OutError = nullptr, std::string_view Default = std::string_view()) const; - - /** Access as a string as UTF8. Checks for range errors and uses the default if OutError is not null. */ - std::u8string_view AsU8String(CbFieldError* OutError = nullptr, std::u8string_view Default = std::u8string_view()) const; - - /** - * Access as an integer, with both positive and negative values returned as unsigned. - * - * Checks for range errors and uses the default if OutError is not null. - */ - uint64_t AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError = nullptr, uint64_t Default = 0) const; - - uint64_t AsIntegerPositive() const; - int64_t AsIntegerNegative() const; - - float AsFloat32() const; - double AsFloat64() const; - - bool AsBool() const; - - inline IoHash AsObjectAttachment() const { return AsHash(); } - inline IoHash AsBinaryAttachment() const { return AsHash(); } - inline IoHash AsAttachment() const { return AsHash(); } - - IoHash AsHash() const; - Guid AsUuid() const; - - int64_t AsDateTimeTicks() const; - int64_t AsTimeSpanTicks() const; - - Oid AsObjectId() const; - - CbCustomById AsCustomById() const; - CbCustomByName AsCustomByName() const; - - inline CbFieldType GetType() const { return Type; } - inline const void* GetData() const { return Data; } - -private: - const void* Data; - CbFieldType Type; -}; - -inline CbFieldView::CbFieldView(const CbValue& InValue) : Type(InValue.GetType()), Payload(InValue.GetData()) -{ -} - -inline CbValue -CbFieldView::GetValue() const -{ - return CbValue(CbFieldTypeOps::GetType(Type), Payload); -} - -inline CbValue::CbValue(CbFieldType InType, const void* InValue) : Data(InValue), Type(InType) -{ -} - -inline CbObjectView -CbValue::AsObjectView() const -{ - return CbObjectView(*this); -} - -inline CbArrayView -CbValue::AsArrayView() const -{ - return CbArrayView(*this); -} - -inline MemoryView -CbValue::AsBinary() const -{ - const uint8_t* const Bytes = static_cast<const uint8_t*>(Data); - uint32_t ValueSizeByteCount; - const uint64_t ValueSize = ReadVarUInt(Bytes, ValueSizeByteCount); - return MakeMemoryView(Bytes + ValueSizeByteCount, ValueSize); -} - -inline std::string_view -CbValue::AsString(CbFieldError* OutError, std::string_view Default) const -{ - const char* const Chars = static_cast<const char*>(Data); - uint32_t ValueSizeByteCount; - const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - - if (OutError) - { - if (ValueSize >= (uint64_t(1) << 31)) - { - *OutError = CbFieldError::RangeError; - return Default; - } - *OutError = CbFieldError::None; - } - - return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); -} - -inline std::u8string_view -CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const -{ - const char8_t* const Chars = static_cast<const char8_t*>(Data); - uint32_t ValueSizeByteCount; - const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - - if (OutError) - { - if (ValueSize >= (uint64_t(1) << 31)) - { - *OutError = CbFieldError::RangeError; - return Default; - } - *OutError = CbFieldError::None; - } - - return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); -} - -inline uint64_t -CbValue::AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError, uint64_t Default) const -{ - // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero. - const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1); - const uint64_t IsNegative = uint8_t(Type) & 1; - - uint32_t MagnitudeByteCount; - const uint64_t Magnitude = ReadVarUInt(Data, MagnitudeByteCount); - const uint64_t Value = Magnitude ^ -int64_t(IsNegative); - - if (OutError) - { - const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned); - *OutError = IsInRange ? CbFieldError::None : CbFieldError::RangeError; - - const uint64_t UseValueMask = -int64_t(IsInRange); - return (Value & UseValueMask) | (Default & ~UseValueMask); - } - - return Value; -} - -inline uint64_t -CbValue::AsIntegerPositive() const -{ - uint32_t MagnitudeByteCount; - return ReadVarUInt(Data, MagnitudeByteCount); -} - -inline int64_t -CbValue::AsIntegerNegative() const -{ - uint32_t MagnitudeByteCount; - return int64_t(ReadVarUInt(Data, MagnitudeByteCount)) ^ -int64_t(1); -} - -inline float -CbValue::AsFloat32() const -{ - const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint32_t>(Data)); - return reinterpret_cast<const float&>(Value); -} - -inline double -CbValue::AsFloat64() const -{ - const uint64_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint64_t>(Data)); - return reinterpret_cast<const double&>(Value); -} - -inline bool -CbValue::AsBool() const -{ - return uint8_t(Type) & 1; -} - -inline IoHash -CbValue::AsHash() const -{ - return IoHash::MakeFrom(Data); -} - -inline Guid -CbValue::AsUuid() const -{ - Guid Value; - memcpy(&Value, Data, sizeof(Guid)); - Value.A = FromNetworkOrder(Value.A); - Value.B = FromNetworkOrder(Value.B); - Value.C = FromNetworkOrder(Value.C); - Value.D = FromNetworkOrder(Value.D); - return Value; -} - -inline int64_t -CbValue::AsDateTimeTicks() const -{ - return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data)); -} - -inline int64_t -CbValue::AsTimeSpanTicks() const -{ - return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data)); -} - -inline Oid -CbValue::AsObjectId() const -{ - return Oid::FromMemory(Data); -} - -inline CbCustomById -CbValue::AsCustomById() const -{ - const uint8_t* Bytes = static_cast<const uint8_t*>(Data); - uint32_t DataSizeByteCount; - const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount); - Bytes += DataSizeByteCount; - - CbCustomById Value; - uint32_t TypeIdByteCount; - Value.Id = ReadVarUInt(Bytes, TypeIdByteCount); - Value.Data = MakeMemoryView(Bytes + TypeIdByteCount, DataSize - TypeIdByteCount); - return Value; -} - -inline CbCustomByName -CbValue::AsCustomByName() const -{ - const uint8_t* Bytes = static_cast<const uint8_t*>(Data); - uint32_t DataSizeByteCount; - const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount); - Bytes += DataSizeByteCount; - - uint32_t TypeNameLenByteCount; - const uint64_t TypeNameLen = ReadVarUInt(Bytes, TypeNameLenByteCount); - Bytes += TypeNameLenByteCount; - - CbCustomByName Value; - Value.Name = std::u8string_view(reinterpret_cast<const char8_t*>(Bytes), static_cast<std::u8string_view::size_type>(TypeNameLen)); - Value.Data = MakeMemoryView(Bytes + TypeNameLen, DataSize - TypeNameLen - TypeNameLenByteCount); - return Value; -} - -} // namespace zen diff --git a/zencore/include/zencore/compositebuffer.h b/zencore/include/zencore/compositebuffer.h deleted file mode 100644 index 4e4b4d002..000000000 --- a/zencore/include/zencore/compositebuffer.h +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/sharedbuffer.h> -#include <zencore/zencore.h> - -#include <functional> -#include <span> -#include <vector> - -namespace zen { - -/** - * CompositeBuffer is a non-contiguous buffer composed of zero or more immutable shared buffers. - * - * A composite buffer is most efficient when its segments are consumed as they are, but it can be - * flattened into a contiguous buffer, when necessary, by calling Flatten(). Ownership of segment - * buffers is not changed on construction, but if ownership of segments is required then that can - * be guaranteed by calling MakeOwned(). - */ - -class CompositeBuffer -{ -public: - /** - * Construct a composite buffer by concatenating the buffers. Does not enforce ownership. - * - * Buffer parameters may be SharedBuffer, CompositeBuffer, or std::vector<SharedBuffer>. - */ - template<typename... BufferTypes> - inline explicit CompositeBuffer(BufferTypes&&... Buffers) - { - if constexpr (sizeof...(Buffers) > 0) - { - m_Segments.reserve((GetBufferCount(std::forward<BufferTypes>(Buffers)) + ...)); - (AppendBuffers(std::forward<BufferTypes>(Buffers)), ...); - std::erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); }); - } - } - - /** Reset this to null. */ - ZENCORE_API void Reset(); - - /** Returns the total size of the composite buffer in bytes. */ - [[nodiscard]] ZENCORE_API uint64_t GetSize() const; - - /** Returns the segments that the buffer is composed from. */ - [[nodiscard]] inline std::span<const SharedBuffer> GetSegments() const { return std::span<const SharedBuffer>{m_Segments}; } - - /** Returns true if the composite buffer is not null. */ - [[nodiscard]] inline explicit operator bool() const { return !IsNull(); } - - /** Returns true if the composite buffer is null. */ - [[nodiscard]] inline bool IsNull() const { return m_Segments.empty(); } - - /** Returns true if every segment in the composite buffer is owned. */ - [[nodiscard]] ZENCORE_API bool IsOwned() const; - - /** Returns a copy of the buffer where every segment is owned. */ - [[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() const&; - [[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() &&; - - /** Returns the concatenation of the segments into a contiguous buffer. */ - [[nodiscard]] ZENCORE_API SharedBuffer Flatten() const&; - [[nodiscard]] ZENCORE_API SharedBuffer Flatten() &&; - - /** Returns the middle part of the buffer by taking the size starting at the offset. */ - [[nodiscard]] ZENCORE_API CompositeBuffer Mid(uint64_t Offset, uint64_t Size = ~uint64_t(0)) const; - - /** - * Returns a view of the range if contained by one segment, otherwise a view of a copy of the range. - * - * @note CopyBuffer is reused if large enough, and otherwise allocated when needed. - * - * @param Offset The byte offset in this buffer that the range starts at. - * @param Size The number of bytes in the range to view or copy. - * @param CopyBuffer The buffer to write the copy into if a copy is required. - */ - [[nodiscard]] ZENCORE_API MemoryView ViewOrCopyRange(uint64_t Offset, uint64_t Size, UniqueBuffer& CopyBuffer) const; - - /** - * Copies a range of the buffer to a contiguous region of memory. - * - * @param Target The view to copy to. Must be no larger than the data available at the offset. - * @param Offset The byte offset in this buffer to start copying from. - */ - ZENCORE_API void CopyTo(MutableMemoryView Target, uint64_t Offset = 0) const; - - /** - * Invokes a visitor with a view of each segment that intersects with a range. - * - * @param Offset The byte offset in this buffer to start visiting from. - * @param Size The number of bytes in the range to visit. - * @param Visitor The visitor to invoke from zero to GetSegments().Num() times. - */ - ZENCORE_API void IterateRange(uint64_t Offset, uint64_t Size, std::function<void(MemoryView View)> Visitor) const; - ZENCORE_API void IterateRange(uint64_t Offset, - uint64_t Size, - std::function<void(MemoryView View, const SharedBuffer& ViewOuter)> Visitor) const; - - struct Iterator - { - size_t SegmentIndex = 0; - uint64_t OffsetInSegment = 0; - }; - ZENCORE_API Iterator GetIterator(uint64_t Offset) const; - ZENCORE_API MemoryView ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const; - ZENCORE_API void CopyTo(MutableMemoryView Target, Iterator& It) const; - - /** A null composite buffer. */ - static const CompositeBuffer Null; - -private: - static inline size_t GetBufferCount(const CompositeBuffer& Buffer) { return Buffer.m_Segments.size(); } - inline void AppendBuffers(const CompositeBuffer& Buffer) - { - m_Segments.insert(m_Segments.end(), begin(Buffer.m_Segments), end(Buffer.m_Segments)); - } - inline void AppendBuffers(CompositeBuffer&& Buffer) - { - // TODO: this operates just like the by-reference version above - m_Segments.insert(m_Segments.end(), begin(Buffer.m_Segments), end(Buffer.m_Segments)); - } - - static inline size_t GetBufferCount(const SharedBuffer&) { return 1; } - inline void AppendBuffers(const SharedBuffer& Buffer) { m_Segments.push_back(Buffer); } - inline void AppendBuffers(SharedBuffer&& Buffer) { m_Segments.push_back(std::move(Buffer)); } - - static inline size_t GetBufferCount(std::vector<SharedBuffer>&& Container) { return Container.size(); } - inline void AppendBuffers(std::vector<SharedBuffer>&& Container) - { - m_Segments.insert(m_Segments.end(), begin(Container), end(Container)); - } - -private: - std::vector<SharedBuffer> m_Segments; -}; - -void compositebuffer_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h deleted file mode 100644 index 99ce20d8a..000000000 --- a/zencore/include/zencore/compress.h +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore/zencore.h" - -#include "zencore/blake3.h" -#include "zencore/compositebuffer.h" - -namespace zen { - -enum class OodleCompressor : uint8_t -{ - NotSet = 0, - Selkie = 1, - Mermaid = 2, - Kraken = 3, - Leviathan = 4, -}; - -enum class OodleCompressionLevel : int8_t -{ - HyperFast4 = -4, - HyperFast3 = -3, - HyperFast2 = -2, - HyperFast1 = -1, - None = 0, - SuperFast = 1, - VeryFast = 2, - Fast = 3, - Normal = 4, - Optimal1 = 5, - Optimal2 = 6, - Optimal3 = 7, - Optimal4 = 8, -}; - -/** - * A compressed buffer stores compressed data in a self-contained format. - * - * A buffer is self-contained in the sense that it can be decompressed without external knowledge - * of the compression format or the size of the raw data. - */ -class CompressedBuffer -{ -public: - /** - * Compress the buffer using the specified compressor and compression level. - * - * Data that does not compress will be return uncompressed, as if with level None. - * - * @note Using a level of None will return a buffer that references owned raw data. - * - * @param RawData The raw data to be compressed. - * @param Compressor The compressor to encode with. May use NotSet if level is None. - * @param CompressionLevel The compression level to encode with. - * @param BlockSize The power-of-two block size to encode raw data in. 0 is default. - * @return An owned compressed buffer, or null on error. - */ - [[nodiscard]] ZENCORE_API static CompressedBuffer Compress(const CompositeBuffer& RawData, - OodleCompressor Compressor = OodleCompressor::Mermaid, - OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, - uint64_t BlockSize = 0); - [[nodiscard]] ZENCORE_API static CompressedBuffer Compress(const SharedBuffer& RawData, - OodleCompressor Compressor = OodleCompressor::Mermaid, - OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, - uint64_t BlockSize = 0); - - /** - * Construct from a compressed buffer previously created by Compress(). - * - * @return A compressed buffer, or null on error, such as an invalid format or corrupt header. - */ - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData); - [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(IoBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(const IoBuffer& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - - /** Reset this to null. */ - inline void Reset() { CompressedData.Reset(); } - - /** Returns true if the compressed buffer is not null. */ - [[nodiscard]] inline explicit operator bool() const { return !IsNull(); } - - /** Returns true if the compressed buffer is null. */ - [[nodiscard]] inline bool IsNull() const { return CompressedData.IsNull(); } - - /** Returns true if the composite buffer is owned. */ - [[nodiscard]] inline bool IsOwned() const { return CompressedData.IsOwned(); } - - /** Returns a copy of the compressed buffer that owns its underlying memory. */ - [[nodiscard]] inline CompressedBuffer MakeOwned() const& { return FromCompressedNoValidate(CompressedData.MakeOwned()); } - [[nodiscard]] inline CompressedBuffer MakeOwned() && { return FromCompressedNoValidate(std::move(CompressedData).MakeOwned()); } - - /** Returns a composite buffer containing the compressed data. May be null. May not be owned. */ - [[nodiscard]] inline const CompositeBuffer& GetCompressed() const& { return CompressedData; } - [[nodiscard]] inline CompositeBuffer GetCompressed() && { return std::move(CompressedData); } - - /** Returns the size of the compressed data. Zero if this is null. */ - [[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); } - - /** Returns the size of the raw data. Zero on error or if this is empty or null. */ - [[nodiscard]] ZENCORE_API uint64_t DecodeRawSize() const; - - /** Returns the hash of the raw data. Zero on error or if this is null. */ - [[nodiscard]] ZENCORE_API IoHash DecodeRawHash() const; - - [[nodiscard]] ZENCORE_API CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const; - - /** - * Returns the compressor and compression level used by this buffer. - * - * The compressor and compression level may differ from those specified when creating the buffer - * because an incompressible buffer is stored with no compression. Parameters cannot be accessed - * if this is null or uses a method other than Oodle, in which case this returns false. - * - * @return True if parameters were written, otherwise false. - */ - [[nodiscard]] ZENCORE_API bool TryGetCompressParameters(OodleCompressor& OutCompressor, - OodleCompressionLevel& OutCompressionLevel, - uint64_t& OutBlockSize) const; - - /** - * Decompress into a memory view that is less or equal GetRawSize() bytes. - */ - [[nodiscard]] ZENCORE_API bool TryDecompressTo(MutableMemoryView RawView, uint64_t RawOffset = 0) const; - - /** - * Decompress into an owned buffer. - * - * @return An owned buffer containing the raw data, or null on error. - */ - [[nodiscard]] ZENCORE_API SharedBuffer Decompress(uint64_t RawOffset = 0, uint64_t RawSize = ~uint64_t(0)) const; - - /** - * Decompress into an owned composite buffer. - * - * @return An owned buffer containing the raw data, or null on error. - */ - [[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const; - - /** A null compressed buffer. */ - static const CompressedBuffer Null; - -private: - CompositeBuffer CompressedData; -}; - -void compress_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/config.h.in b/zencore/include/zencore/config.h.in deleted file mode 100644 index 3372eca2a..000000000 --- a/zencore/include/zencore/config.h.in +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -// NOTE: Generated from config.h.in - -#define ZEN_CFG_VERSION "${VERSION}" -#define ZEN_CFG_VERSION_MAJOR ${VERSION_MAJOR} -#define ZEN_CFG_VERSION_MINOR ${VERSION_MINOR} -#define ZEN_CFG_VERSION_ALTER ${VERSION_ALTER} -#define ZEN_CFG_VERSION_BUILD ${VERSION_BUILD} -#define ZEN_CFG_VERSION_BRANCH "${GIT_BRANCH}" -#define ZEN_CFG_VERSION_COMMIT "${GIT_COMMIT}" -#define ZEN_CFG_VERSION_BUILD_STRING "${VERSION}-${plat}-${arch}-${mode}" -#define ZEN_CFG_VERSION_BUILD_STRING_FULL "${VERSION}-${VERSION_BUILD}-${plat}-${arch}-${mode}-${GIT_COMMIT}" -#define ZEN_CFG_SCHEMA_VERSION ${ZEN_SCHEMA_VERSION} diff --git a/zencore/include/zencore/crc32.h b/zencore/include/zencore/crc32.h deleted file mode 100644 index 336bda77e..000000000 --- a/zencore/include/zencore/crc32.h +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -namespace zen { - -uint32_t MemCrc32(const void* InData, size_t Length, uint32_t Crc = 0); -uint32_t MemCrc32_Deprecated(const void* InData, size_t Length, uint32_t Crc = 0); -uint32_t StrCrc_Deprecated(const char* Data); - -} // namespace zen diff --git a/zencore/include/zencore/crypto.h b/zencore/include/zencore/crypto.h deleted file mode 100644 index 83d416b0f..000000000 --- a/zencore/include/zencore/crypto.h +++ /dev/null @@ -1,77 +0,0 @@ - -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/memory.h> -#include <zencore/zencore.h> - -#include <memory> -#include <optional> - -namespace zen { - -template<size_t BitCount> -struct CryptoBits -{ -public: - static constexpr size_t ByteCount = BitCount / 8; - - CryptoBits() = default; - - bool IsNull() const { return memcmp(&m_Bits, &Zero, ByteCount) == 0; } - bool IsValid() const { return IsNull() == false; } - - size_t GetSize() const { return ByteCount; } - size_t GetBitCount() const { return BitCount; } - - MemoryView GetView() const { return MemoryView(m_Bits, ByteCount); } - - static CryptoBits FromMemoryView(MemoryView Bits) - { - if (Bits.GetSize() != ByteCount) - { - return CryptoBits(); - } - - return CryptoBits(Bits); - } - - static CryptoBits FromString(std::string_view Str) { return FromMemoryView(MakeMemoryView(Str)); } - -private: - CryptoBits(MemoryView Bits) - { - ZEN_ASSERT(Bits.GetSize() == GetSize()); - memcpy(&m_Bits, Bits.GetData(), GetSize()); - } - - static constexpr uint8_t Zero[ByteCount] = {0}; - - uint8_t m_Bits[ByteCount] = {0}; -}; - -using AesKey256Bit = CryptoBits<256>; -using AesIV128Bit = CryptoBits<128>; - -class Aes -{ -public: - static constexpr size_t BlockSize = 16; - - static MemoryView Encrypt(const AesKey256Bit& Key, - const AesIV128Bit& IV, - MemoryView In, - MutableMemoryView Out, - std::optional<std::string>& Reason); - - static MemoryView Decrypt(const AesKey256Bit& Key, - const AesIV128Bit& IV, - MemoryView In, - MutableMemoryView Out, - std::optional<std::string>& Reason); -}; - -void crypto_forcelink(); - -} // namespace zen diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h deleted file mode 100644 index 7a9e6b44c..000000000 --- a/zencore/include/zencore/endian.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <cstdint> - -namespace zen { - -inline uint16_t -ByteSwap(uint16_t x) -{ -#if ZEN_COMPILER_MSC - return _byteswap_ushort(x); -#else - return __builtin_bswap16(x); -#endif -} - -inline uint32_t -ByteSwap(uint32_t x) -{ -#if ZEN_COMPILER_MSC - return _byteswap_ulong(x); -#else - return __builtin_bswap32(x); -#endif -} - -inline uint64_t -ByteSwap(uint64_t x) -{ -#if ZEN_COMPILER_MSC - return _byteswap_uint64(x); -#else - return __builtin_bswap64(x); -#endif -} - -inline uint16_t -FromNetworkOrder(uint16_t x) -{ - return ByteSwap(x); -} - -inline uint32_t -FromNetworkOrder(uint32_t x) -{ - return ByteSwap(x); -} - -inline uint64_t -FromNetworkOrder(uint64_t x) -{ - return ByteSwap(x); -} - -inline uint16_t -FromNetworkOrder(int16_t x) -{ - return ByteSwap(uint16_t(x)); -} - -inline uint32_t -FromNetworkOrder(int32_t x) -{ - return ByteSwap(uint32_t(x)); -} - -inline uint64_t -FromNetworkOrder(int64_t x) -{ - return ByteSwap(uint64_t(x)); -} - -inline uint16_t -ToNetworkOrder(uint16_t x) -{ - return ByteSwap(x); -} - -inline uint32_t -ToNetworkOrder(uint32_t x) -{ - return ByteSwap(x); -} - -inline uint64_t -ToNetworkOrder(uint64_t x) -{ - return ByteSwap(x); -} - -inline uint16_t -ToNetworkOrder(int16_t x) -{ - return ByteSwap(uint16_t(x)); -} - -inline uint32_t -ToNetworkOrder(int32_t x) -{ - return ByteSwap(uint32_t(x)); -} - -inline uint64_t -ToNetworkOrder(int64_t x) -{ - return ByteSwap(uint64_t(x)); -} - -} // namespace zen diff --git a/zencore/include/zencore/enumflags.h b/zencore/include/zencore/enumflags.h deleted file mode 100644 index ebe747bf0..000000000 --- a/zencore/include/zencore/enumflags.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -namespace zen { - -// Enum class helpers - -// Defines all bitwise operators for enum classes so it can be (mostly) used as a regular flags enum -#define ENUM_CLASS_FLAGS(Enum) \ - inline Enum& operator|=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \ - inline Enum& operator&=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \ - inline Enum& operator^=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \ - inline constexpr Enum operator|(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \ - inline constexpr Enum operator&(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \ - inline constexpr Enum operator^(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \ - inline constexpr bool operator!(Enum E) { return !(__underlying_type(Enum))E; } \ - inline constexpr Enum operator~(Enum E) { return (Enum) ~(__underlying_type(Enum))E; } - -// Friends all bitwise operators for enum classes so the definition can be kept private / protected. -#define FRIEND_ENUM_CLASS_FLAGS(Enum) \ - friend Enum& operator|=(Enum& Lhs, Enum Rhs); \ - friend Enum& operator&=(Enum& Lhs, Enum Rhs); \ - friend Enum& operator^=(Enum& Lhs, Enum Rhs); \ - friend constexpr Enum operator|(Enum Lhs, Enum Rhs); \ - friend constexpr Enum operator&(Enum Lhs, Enum Rhs); \ - friend constexpr Enum operator^(Enum Lhs, Enum Rhs); \ - friend constexpr bool operator!(Enum E); \ - friend constexpr Enum operator~(Enum E); - -template<typename Enum> -constexpr bool -EnumHasAllFlags(Enum Flags, Enum Contains) -{ - return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) == ((__underlying_type(Enum))Contains); -} - -template<typename Enum> -constexpr bool -EnumHasAnyFlags(Enum Flags, Enum Contains) -{ - return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) != 0; -} - -template<typename Enum> -void -EnumAddFlags(Enum& Flags, Enum FlagsToAdd) -{ - Flags |= FlagsToAdd; -} - -template<typename Enum> -void -EnumRemoveFlags(Enum& Flags, Enum FlagsToRemove) -{ - Flags &= ~FlagsToRemove; -} - -} // namespace zen diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h deleted file mode 100644 index c61db5ba9..000000000 --- a/zencore/include/zencore/except.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/string.h> -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#else -# include <errno.h> -#endif -#if __has_include("source_location") -# include <source_location> -#endif -#include <string> -#include <system_error> - -namespace zen { - -#if ZEN_PLATFORM_WINDOWS -ZENCORE_API void ThrowSystemException [[noreturn]] (HRESULT hRes, std::string_view Message); -#endif // ZEN_PLATFORM_WINDOWS - -#if defined(__cpp_lib_source_location) -ZENCORE_API void ThrowLastErrorImpl [[noreturn]] (std::string_view Message, const std::source_location& Location); -# define ThrowLastError(Message) ThrowLastErrorImpl(Message, std::source_location::current()) -#else -ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message); -#endif - -ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); - -ZENCORE_API std::string GetLastErrorAsString(); -ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode); - -inline int32_t -GetLastError() -{ -#if ZEN_PLATFORM_WINDOWS - return ::GetLastError(); -#else - return errno; -#endif -} - -inline std::error_code -MakeErrorCode(uint32_t ErrorCode) noexcept -{ - return std::error_code(ErrorCode, std::system_category()); -} - -inline std::error_code -MakeErrorCodeFromLastError() noexcept -{ - return std::error_code(zen::GetLastError(), std::system_category()); -} - -} // namespace zen diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h deleted file mode 100644 index fa5f94170..000000000 --- a/zencore/include/zencore/filesystem.h +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <zencore/iobuffer.h> -#include <zencore/string.h> - -#include <filesystem> -#include <functional> - -namespace zen { - -class IoBuffer; - -/** Delete directory (after deleting any contents) - */ -ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); - -/** Ensure directory exists. - - Will also create any required parent directories - */ -ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); - -/** Ensure directory exists and delete contents (if any) before returning - */ -ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); - -/** Map native file handle to a path - */ -ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle); - -ZENCORE_API std::filesystem::path GetRunningExecutablePath(); - -/** Set the max open file handle count to max allowed for the current process on Linux and MacOS - */ -ZENCORE_API void MaximizeOpenFileCount(); - -struct FileContents -{ - std::vector<IoBuffer> Data; - std::error_code ErrorCode; - - IoBuffer Flatten(); -}; - -ZENCORE_API FileContents ReadStdIn(); -ZENCORE_API FileContents ReadFile(std::filesystem::path Path); -ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc); -ZENCORE_API void WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount); -ZENCORE_API void WriteFile(std::filesystem::path Path, IoBuffer Data); - -struct CopyFileOptions -{ - bool EnableClone = true; - bool MustClone = false; -}; - -ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); -ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); - -ZENCORE_API void PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out); -ZENCORE_API std::string PathToUtf8(const std::filesystem::path& Path); - -extern template class StringBuilderImpl<std::filesystem::path::value_type>; - -/** - * Helper class for building paths. Backed by a string builder. - * - */ -class PathBuilderBase : public StringBuilderImpl<std::filesystem::path::value_type> -{ -private: - using Super = StringBuilderImpl<std::filesystem::path::value_type>; - -protected: - using CharType = std::filesystem::path::value_type; - using ViewType = std::basic_string_view<CharType>; - -public: - void Append(const std::filesystem::path& Rhs) { Super::Append(Rhs.c_str()); } - void operator/=(const std::filesystem::path& Rhs) { this->operator/=(Rhs.c_str()); }; - void operator/=(const CharType* Rhs) - { - AppendSeparator(); - Super::Append(Rhs); - } - operator ViewType() const { return ToView(); } - std::basic_string_view<CharType> ToView() const { return std::basic_string_view<CharType>(Data(), Size()); } - std::filesystem::path ToPath() const { return std::filesystem::path(ToView()); } - - std::string ToUtf8() const - { -#if ZEN_PLATFORM_WINDOWS - return WideToUtf8(ToView()); -#else - return std::string(ToView()); -#endif - } - - void AppendSeparator() - { - if (ToView().ends_with(std::filesystem::path::preferred_separator) -#if ZEN_PLATFORM_WINDOWS - || ToView().ends_with('/') -#endif - ) - return; - - Super::Append(std::filesystem::path::preferred_separator); - } -}; - -template<size_t N> -class PathBuilder : public PathBuilderBase -{ -public: - PathBuilder() { Init(m_Buffer, N); } - -private: - PathBuilderBase::CharType m_Buffer[N]; -}; - -template<size_t N> -class ExtendablePathBuilder : public PathBuilder<N> -{ -public: - ExtendablePathBuilder() { this->m_IsExtendable = true; } -}; - -struct DiskSpace -{ - uint64_t Free{}; - uint64_t Total{}; -}; - -ZENCORE_API DiskSpace DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error); - -inline bool -DiskSpaceInfo(std::filesystem::path Directory, DiskSpace& Space) -{ - std::error_code Err; - Space = DiskSpaceInfo(Directory, Err); - return !Err; -} - -/** - * Efficient file system traversal - * - * Uses the best available mechanism for the platform in question and could take - * advantage of any file system tracking mechanisms in the future - * - */ -class FileSystemTraversal -{ -public: - struct TreeVisitor - { - using path_view = std::basic_string_view<std::filesystem::path::value_type>; - using path_string = std::filesystem::path::string_type; - - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0; - - // This should return true if we should recurse into the directory - virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0; - }; - - void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor); -}; - -struct DirectoryContent -{ - static const uint8_t IncludeDirsFlag = 1u << 0; - static const uint8_t IncludeFilesFlag = 1u << 1; - static const uint8_t RecursiveFlag = 1u << 2; - std::vector<std::filesystem::path> Files; - std::vector<std::filesystem::path> Directories; -}; - -void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent); - -std::string GetEnvVariable(std::string_view VariableName); - -////////////////////////////////////////////////////////////////////////// - -void filesystem_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/fmtutils.h b/zencore/include/zencore/fmtutils.h deleted file mode 100644 index 70867fe72..000000000 --- a/zencore/include/zencore/fmtutils.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/iohash.h> -#include <zencore/string.h> -#include <zencore/uid.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <fmt/format.h> -ZEN_THIRD_PARTY_INCLUDES_END - -#include <filesystem> -#include <string_view> - -// Custom formatting for some zencore types - -template<> -struct fmt::formatter<zen::IoHash> : formatter<string_view> -{ - template<typename FormatContext> - auto format(const zen::IoHash& Hash, FormatContext& ctx) - { - zen::IoHash::String_t String; - Hash.ToHexString(String); - return formatter<string_view>::format({String, zen::IoHash::StringLength}, ctx); - } -}; - -template<> -struct fmt::formatter<zen::Oid> : formatter<string_view> -{ - template<typename FormatContext> - auto format(const zen::Oid& Id, FormatContext& ctx) - { - zen::StringBuilder<32> String; - Id.ToString(String); - return formatter<string_view>::format({String.c_str(), zen::Oid::StringLength}, ctx); - } -}; - -template<> -struct fmt::formatter<std::filesystem::path> : formatter<string_view> -{ - template<typename FormatContext> - auto format(const std::filesystem::path& Path, FormatContext& ctx) - { - zen::ExtendableStringBuilder<128> String; - String << Path.u8string(); - return formatter<string_view>::format(String.ToView(), ctx); - } -}; diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h deleted file mode 100644 index f24caed6e..000000000 --- a/zencore/include/zencore/intmath.h +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <stdint.h> - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_COMPILER_MSC || ZEN_PLATFORM_WINDOWS -# pragma intrinsic(_BitScanReverse) -# pragma intrinsic(_BitScanReverse64) -#else -inline uint8_t -_BitScanReverse(unsigned long* Index, uint32_t Mask) -{ - if (Mask == 0) - { - return 0; - } - - *Index = 31 - __builtin_clz(Mask); - return 1; -} - -inline uint8_t -_BitScanReverse64(unsigned long* Index, uint64_t Mask) -{ - if (Mask == 0) - { - return 0; - } - - *Index = 63 - __builtin_clzll(Mask); - return 1; -} - -inline uint8_t -_BitScanForward64(unsigned long* Index, uint64_t Mask) -{ - if (Mask == 0) - { - return 0; - } - - *Index = __builtin_ctzll(Mask); - return 1; -} -#endif - -namespace zen { - -inline constexpr bool -IsPow2(uint64_t n) -{ - return 0 == (n & (n - 1)); -} - -/// Round an integer up to the closest integer multiplier of 'base' ('base' must be a power of two) -template<Integral T> -T -RoundUp(T Value, auto Base) -{ - ZEN_ASSERT_SLOW(IsPow2(Base)); - return ((Value + T(Base - 1)) & (~T(Base - 1))); -} - -bool -IsMultipleOf(Integral auto Value, auto MultiplierPow2) -{ - ZEN_ASSERT_SLOW(IsPow2(MultiplierPow2)); - return (Value & (MultiplierPow2 - 1)) == 0; -} - -inline uint64_t -NextPow2(uint64_t n) -{ - // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 - - --n; - - n |= n >> 1; - n |= n >> 2; - n |= n >> 4; - n |= n >> 8; - n |= n >> 16; - n |= n >> 32; - - return n + 1; -} - -static inline uint32_t -FloorLog2(uint32_t Value) -{ - // Use BSR to return the log2 of the integer - unsigned long Log2; - if (_BitScanReverse(&Log2, Value) != 0) - { - return Log2; - } - - return 0; -} - -static inline uint32_t -CountLeadingZeros(uint32_t Value) -{ - unsigned long Log2 = 0; - _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1); - return 32 - Log2; -} - -static inline uint64_t -FloorLog2_64(uint64_t Value) -{ - unsigned long Log2 = 0; - long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); - return Log2 & Mask; -} - -static inline uint64_t -CountLeadingZeros64(uint64_t Value) -{ - unsigned long Log2 = 0; - long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); - return ((63 - Log2) & Mask) | (64 & ~Mask); -} - -static inline uint64_t -CeilLogTwo64(uint64_t Arg) -{ - int64_t Bitmask = ((int64_t)(CountLeadingZeros64(Arg) << 57)) >> 63; - return (64 - CountLeadingZeros64(Arg - 1)) & (~Bitmask); -} - -static inline uint64_t -CountTrailingZeros64(uint64_t Value) -{ - if (Value == 0) - { - return 64; - } - unsigned long BitIndex; // 0-based, where the LSB is 0 and MSB is 31 - _BitScanForward64(&BitIndex, Value); // Scans from LSB to MSB - return BitIndex; -} - -////////////////////////////////////////////////////////////////////////// - -static inline bool -IsPointerAligned(const void* Ptr, uint64_t Alignment) -{ - ZEN_ASSERT_SLOW(IsPow2(Alignment)); - - return 0 == (reinterpret_cast<uintptr_t>(Ptr) & (Alignment - 1)); -} - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_PLATFORM_WINDOWS -# ifdef min -# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead" -# endif -#endif - -constexpr auto -Min(auto x, auto y) -{ - return x < y ? x : y; -} - -constexpr auto -Max(auto x, auto y) -{ - return x > y ? x : y; -} - -////////////////////////////////////////////////////////////////////////// - -void intmath_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h deleted file mode 100644 index a39dbf6d6..000000000 --- a/zencore/include/zencore/iobuffer.h +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <memory.h> -#include <zencore/memory.h> -#include <atomic> -#include "refcount.h" -#include "zencore.h" - -#include <filesystem> - -namespace zen { - -struct IoHash; -struct IoBufferExtendedCore; - -enum class ZenContentType : uint8_t -{ - kBinary = 0, // Note that since this is zero, this will be the default value in IoBuffer - kText = 1, - kJSON = 2, - kCbObject = 3, - kCbPackage = 4, - kYAML = 5, - kCbPackageOffer = 6, - kCompressedBinary = 7, - kUnknownContentType = 8, - kHTML = 9, - kJavaScript = 10, - kCSS = 11, - kPNG = 12, - kIcon = 13, - kCOUNT -}; - -inline std::string_view -ToString(ZenContentType ContentType) -{ - using namespace std::literals; - - switch (ContentType) - { - default: - case ZenContentType::kUnknownContentType: - return "unknown"sv; - case ZenContentType::kBinary: - return "binary"sv; - case ZenContentType::kText: - return "text"sv; - case ZenContentType::kJSON: - return "json"sv; - case ZenContentType::kCbObject: - return "cb-object"sv; - case ZenContentType::kCbPackage: - return "cb-package"sv; - case ZenContentType::kCbPackageOffer: - return "cb-package-offer"sv; - case ZenContentType::kCompressedBinary: - return "compressed-binary"sv; - case ZenContentType::kYAML: - return "yaml"sv; - case ZenContentType::kHTML: - return "html"sv; - case ZenContentType::kJavaScript: - return "javascript"sv; - case ZenContentType::kCSS: - return "css"sv; - case ZenContentType::kPNG: - return "png"sv; - case ZenContentType::kIcon: - return "icon"sv; - } -} - -struct IoBufferFileReference -{ - void* FileHandle; - uint64_t FileChunkOffset; - uint64_t FileChunkSize; -}; - -struct IoBufferCore -{ -public: - inline IoBufferCore() : m_Flags(kIsNull) {} - inline IoBufferCore(const void* DataPtr, size_t SizeBytes) : m_DataPtr(DataPtr), m_DataBytes(SizeBytes) {} - inline IoBufferCore(const IoBufferCore* Outer, const void* DataPtr, size_t SizeBytes) - : m_DataPtr(DataPtr) - , m_DataBytes(SizeBytes) - , m_OuterCore(Outer) - { - } - - ZENCORE_API explicit IoBufferCore(size_t SizeBytes); - ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment); - ZENCORE_API ~IoBufferCore(); - - // Reference counting - - inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); } - inline uint32_t Release() const - { - const uint32_t NewRefCount = AtomicDecrement(const_cast<IoBufferCore*>(this)->m_RefCount); - if (NewRefCount == 0) - { - DeleteThis(); - } - return NewRefCount; - } - - // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it - - IoBufferCore(const IoBufferCore&) = delete; - IoBufferCore(IoBufferCore&&) = delete; - IoBufferCore& operator=(const IoBufferCore&) = delete; - IoBufferCore& operator=(IoBufferCore&&) = delete; - - // - - ZENCORE_API void Materialize() const; - ZENCORE_API void DeleteThis() const; - ZENCORE_API void MakeOwned(bool Immutable = true); - - inline void EnsureDataValid() const - { - const uint32_t LocalFlags = m_Flags.load(std::memory_order_acquire); - if ((LocalFlags & kIsExtended) && !(LocalFlags & kIsMaterialized)) - { - Materialize(); - } - } - - inline bool IsOwnedByThis() const { return !!(m_Flags.load(std::memory_order_relaxed) & kIsOwnedByThis); } - - inline void SetIsOwnedByThis(bool NewState) - { - if (NewState) - { - m_Flags.fetch_or(kIsOwnedByThis, std::memory_order_relaxed); - } - else - { - m_Flags.fetch_and(~kIsOwnedByThis, std::memory_order_relaxed); - } - } - - inline bool IsOwned() const - { - if (IsOwnedByThis()) - { - return true; - } - return m_OuterCore && m_OuterCore->IsOwned(); - } - - inline bool IsImmutable() const { return (m_Flags.load(std::memory_order_relaxed) & kIsMutable) == 0; } - inline bool IsWholeFile() const { return (m_Flags.load(std::memory_order_relaxed) & kIsWholeFile) != 0; } - inline bool IsNull() const { return (m_Flags.load(std::memory_order_relaxed) & kIsNull) != 0; } - - inline IoBufferExtendedCore* ExtendedCore(); - inline const IoBufferExtendedCore* ExtendedCore() const; - - ZENCORE_API void* MutableDataPointer() const; - - inline const void* DataPointer() const - { - EnsureDataValid(); - return m_DataPtr; - } - - inline size_t DataBytes() const { return m_DataBytes; } - - inline void Set(const void* Ptr, size_t Sz) - { - m_DataPtr = Ptr; - m_DataBytes = Sz; - } - - inline void SetIsImmutable(bool NewState) - { - if (!NewState) - { - m_Flags.fetch_or(kIsMutable, std::memory_order_relaxed); - } - else - { - m_Flags.fetch_and(~kIsMutable, std::memory_order_relaxed); - } - } - - inline void SetIsWholeFile(bool NewState) - { - if (NewState) - { - m_Flags.fetch_or(kIsWholeFile, std::memory_order_relaxed); - } - else - { - m_Flags.fetch_and(~kIsWholeFile, std::memory_order_relaxed); - } - } - - inline void SetContentType(ZenContentType ContentType) - { - ZEN_ASSERT_SLOW((uint32_t(ContentType) & kContentTypeMask) == uint32_t(ContentType)); - uint32_t OldValue = m_Flags.load(std::memory_order_relaxed); - uint32_t NewValue; - do - { - NewValue = (OldValue & ~(kContentTypeMask << kContentTypeShift)) | (uint32_t(ContentType) << kContentTypeShift); - } while (!m_Flags.compare_exchange_weak(OldValue, NewValue, std::memory_order_relaxed, std::memory_order_relaxed)); - } - - inline ZenContentType GetContentType() const - { - return ZenContentType((m_Flags.load(std::memory_order_relaxed) >> kContentTypeShift) & kContentTypeMask); - } - - inline uint32_t GetRefCount() const { return m_RefCount; } - -protected: - uint32_t m_RefCount = 0; - mutable std::atomic<uint32_t> m_Flags{0}; - mutable const void* m_DataPtr = nullptr; - size_t m_DataBytes = 0; - RefPtr<const IoBufferCore> m_OuterCore; - - enum - { - kContentTypeShift = 24, - kContentTypeMask = 0xf - }; - - static_assert((uint32_t(ZenContentType::kUnknownContentType) & ~kContentTypeMask) == 0); - - enum Flags : uint32_t - { - kIsNull = 1 << 0, // This is a null IoBuffer - kIsMutable = 1 << 1, - kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore - kIsMaterialized = 1 << 3, // Data pointers are valid - kLowLevelAlloc = 1 << 4, // Using direct memory allocation - kIsWholeFile = 1 << 5, // References an entire file - kIoBufferAlloc = 1 << 6, // Using IoBuffer allocator - kIsOwnedByThis = 1 << 7, - - // Note that we have some extended flags defined below - // so not all bits are available to use here - - kContentTypeBit0 = 1 << (24 + 0), // These constants - kContentTypeBit1 = 1 << (24 + 1), // are here mostly to - kContentTypeBit2 = 1 << (24 + 2), // indicate that these - kContentTypeBit3 = 1 << (24 + 3), // bits are reserved - }; - - void AllocateBuffer(size_t InSize, size_t Alignment) const; - void FreeBuffer(); -}; - -/** - * An "Extended" core references a segment of a file - */ - -struct IoBufferExtendedCore : public IoBufferCore -{ - IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership); - IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size); - ~IoBufferExtendedCore(); - - enum ExtendedFlags - { - kOwnsFile = 1 << 16, - kOwnsMmap = 1 << 17 - }; - - void Materialize() const; - bool GetFileReference(IoBufferFileReference& OutRef) const; - void MarkAsDeleteOnClose(); - -private: - void* m_FileHandle = nullptr; - uint64_t m_FileOffset = 0; - mutable void* m_MmapHandle = nullptr; - mutable void* m_MappedPointer = nullptr; - bool m_DeleteOnClose = false; -}; - -inline IoBufferExtendedCore* -IoBufferCore::ExtendedCore() -{ - if (m_Flags.load(std::memory_order_relaxed) & kIsExtended) - { - return static_cast<IoBufferExtendedCore*>(this); - } - - return nullptr; -} - -inline const IoBufferExtendedCore* -IoBufferCore::ExtendedCore() const -{ - if (m_Flags.load(std::memory_order_relaxed) & kIsExtended) - { - return static_cast<const IoBufferExtendedCore*>(this); - } - - return nullptr; -} - -/** - * I/O buffer - * - * This represents a reference to a payload in memory or on disk - * - */ -class IoBuffer -{ -public: - enum ECloneTag - { - Clone - }; - enum EWrapTag - { - Wrap - }; - enum EFileTag - { - File - }; - enum EBorrowedFileTag - { - BorrowedFile - }; - - inline IoBuffer() = default; - inline IoBuffer(IoBuffer&& Rhs) noexcept = default; - inline IoBuffer(const IoBuffer& Rhs) = default; - inline IoBuffer& operator=(const IoBuffer& Rhs) = default; - inline IoBuffer& operator=(IoBuffer&& Rhs) noexcept = default; - - /** Create an uninitialized buffer of the given size - */ - ZENCORE_API explicit IoBuffer(size_t InSize); - - /** Create an uninitialized buffer of the given size with the specified alignment - */ - ZENCORE_API explicit IoBuffer(size_t InSize, uint64_t InAlignment); - - /** Create a buffer which references a sequence of bytes inside another buffer - */ - ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes = ~0ull); - - /** Create a buffer which references a range of bytes which we assume will live - * for the entire life time. - */ - inline IoBuffer(EWrapTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(DataPtr, SizeBytes)) {} - - inline IoBuffer(ECloneTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(SizeBytes)) - { - memcpy(const_cast<void*>(m_Core->DataPointer()), DataPtr, SizeBytes); - } - - ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); - ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); - - inline explicit operator bool() const { return !m_Core->IsNull(); } - inline operator MemoryView() const& { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); } - inline void MakeOwned() { return m_Core->MakeOwned(); } - [[nodiscard]] inline bool IsOwned() const { return m_Core->IsOwned(); } - [[nodiscard]] inline bool IsWholeFile() const { return m_Core->IsWholeFile(); } - [[nodiscard]] void* MutableData() const { return m_Core->MutableDataPointer(); } - void MakeImmutable() { m_Core->SetIsImmutable(true); } - [[nodiscard]] const void* Data() const { return m_Core->DataPointer(); } - [[nodiscard]] const void* GetData() const { return m_Core->DataPointer(); } - [[nodiscard]] size_t Size() const { return m_Core->DataBytes(); } - [[nodiscard]] size_t GetSize() const { return m_Core->DataBytes(); } - inline void SetContentType(ZenContentType ContentType) { m_Core->SetContentType(ContentType); } - [[nodiscard]] inline ZenContentType GetContentType() const { return m_Core->GetContentType(); } - [[nodiscard]] ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const; - void MarkAsDeleteOnClose(); - - inline MemoryView GetView() const { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); } - inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Core->MutableDataPointer(), m_Core->DataBytes()); } - - template<typename T> - [[nodiscard]] const T* Data() const - { - return reinterpret_cast<const T*>(m_Core->DataPointer()); - } - - template<typename T> - [[nodiscard]] T* MutableData() const - { - return reinterpret_cast<T*>(m_Core->MutableDataPointer()); - } - -private: - RefPtr<IoBufferCore> m_Core = new IoBufferCore; - - IoBuffer(IoBufferCore* Core) : m_Core(Core) {} - - friend class SharedBuffer; - friend class IoBufferBuilder; -}; - -class IoBufferBuilder -{ -public: - ZENCORE_API static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); - ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); - ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); - ZENCORE_API static IoBuffer ReadFromFileMaybe(IoBuffer& InBuffer); - inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); } - inline static IoBuffer MakeCloneFromMemory(MemoryView Memory) { return IoBuffer(IoBuffer::Clone, Memory.GetData(), Memory.GetSize()); } -}; - -IoHash HashBuffer(IoBuffer& Buffer); - -void iobuffer_forcelink(); - -} // namespace zen diff --git a/zencore/include/zencore/iohash.h b/zencore/include/zencore/iohash.h deleted file mode 100644 index fd0f4b2a7..000000000 --- a/zencore/include/zencore/iohash.h +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <zencore/blake3.h> -#include <zencore/memory.h> - -#include <compare> -#include <string_view> - -namespace zen { - -class StringBuilderBase; -class CompositeBuffer; - -/** - * Hash used for content addressable storage - * - * This is basically a BLAKE3-160 hash (note: this is probably not an officially - * recognized identifier). It is generated by computing a 32-byte BLAKE3 hash and - * picking the first 20 bytes of the resulting hash. - * - */ -struct IoHash -{ - alignas(uint32_t) uint8_t Hash[20] = {}; - - static IoHash MakeFrom(const void* data /* 20 bytes */) - { - IoHash Io; - memcpy(Io.Hash, data, sizeof Io); - return Io; - } - - static IoHash FromBLAKE3(const BLAKE3& Blake3) - { - IoHash Io; - memcpy(Io.Hash, Blake3.Hash, sizeof Io.Hash); - return Io; - } - - static IoHash HashBuffer(const void* data, size_t byteCount); - static IoHash HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); } - static IoHash HashBuffer(const CompositeBuffer& Buffer); - static IoHash FromHexString(const char* string); - static IoHash FromHexString(const std::string_view string); - const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; - StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; - std::string ToHexString() const; - - static const int StringLength = 40; - typedef char String_t[StringLength + 1]; - - static const IoHash Zero; // Initialized to all zeros - - inline auto operator<=>(const IoHash& rhs) const = default; - - struct Hasher - { - size_t operator()(const IoHash& v) const - { - size_t h; - memcpy(&h, v.Hash, sizeof h); - return h; - } - }; -}; - -struct IoHashStream -{ - /// Begin streaming hash compute (not needed on freshly constructed instance) - void Reset() { m_Blake3Stream.Reset(); } - - /// Append another chunk - IoHashStream& Append(const void* data, size_t byteCount) - { - m_Blake3Stream.Append(data, byteCount); - return *this; - } - - /// Append another chunk - IoHashStream& Append(MemoryView Data) - { - m_Blake3Stream.Append(Data.GetData(), Data.GetSize()); - return *this; - } - - /// Obtain final hash. If you wish to reuse the instance call reset() - IoHash GetHash() - { - BLAKE3 b3 = m_Blake3Stream.GetHash(); - - IoHash Io; - memcpy(Io.Hash, b3.Hash, sizeof Io.Hash); - - return Io; - } - -private: - BLAKE3Stream m_Blake3Stream; -}; - -} // namespace zen - -namespace std { - -template<> -struct hash<zen::IoHash> : public zen::IoHash::Hasher -{ -}; - -} // namespace std diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h deleted file mode 100644 index 5cbe034cf..000000000 --- a/zencore/include/zencore/logging.h +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <spdlog/spdlog.h> -#undef GetObject -ZEN_THIRD_PARTY_INCLUDES_END - -#include <string_view> - -namespace zen::logging { - -spdlog::logger& Default(); -void SetDefault(std::shared_ptr<spdlog::logger> NewDefaultLogger); -spdlog::logger& ConsoleLog(); -spdlog::logger& Get(std::string_view Name); -spdlog::logger* ErrorLog(); -void SetErrorLog(std::shared_ptr<spdlog::logger>&& NewErrorLogger); - -void InitializeLogging(); -void ShutdownLogging(); - -} // namespace zen::logging - -namespace zen { -extern spdlog::logger* TheDefaultLogger; - -inline spdlog::logger& -Log() -{ - return *TheDefaultLogger; -} - -using logging::ConsoleLog; -using logging::ErrorLog; -} // namespace zen - -using zen::ConsoleLog; -using zen::ErrorLog; -using zen::Log; - -struct LogCategory -{ - LogCategory(std::string_view InCategory) : Category(InCategory) {} - - spdlog::logger& Logger() - { - static spdlog::logger& Inst = zen::logging::Get(Category); - return Inst; - } - - std::string Category; -}; - -inline consteval bool -LogIsErrorLevel(int level) -{ - return (level == spdlog::level::err || level == spdlog::level::critical); -}; - -#define ZEN_LOG_WITH_LOCATION(logger, loc, level, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - if (logger.should_log(level)) \ - { \ - logger.log(loc, level, fmtstr, ##__VA_ARGS__); \ - if (LogIsErrorLevel(level)) \ - { \ - if (auto ErrLogger = zen::logging::ErrorLog(); ErrLogger != nullptr) \ - { \ - ErrLogger->log(loc, level, fmtstr, ##__VA_ARGS__); \ - } \ - } \ - } \ - } while (false); - -#define ZEN_LOG(logger, level, fmtstr, ...) ZEN_LOG_WITH_LOCATION(logger, spdlog::source_loc{}, level, fmtstr, ##__VA_ARGS__) - -#define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \ - static struct LogCategory##Category : public LogCategory \ - { \ - LogCategory##Category() : LogCategory(Name) {} \ - } Category; - -#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::trace, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::debug, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::info, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), spdlog::level::warn, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_LOG_ERROR(Category, fmtstr, ...) \ - ZEN_LOG_WITH_LOCATION(Category.Logger(), \ - spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \ - spdlog::level::err, \ - fmtstr##sv, \ - ##__VA_ARGS__) - -#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) \ - ZEN_LOG_WITH_LOCATION(Category.Logger(), \ - spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \ - spdlog::level::critical, \ - fmtstr##sv, \ - ##__VA_ARGS__) - - // Helper macros for logging - -#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::trace, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::debug, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::info, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), spdlog::level::warn, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_ERROR(fmtstr, ...) \ - ZEN_LOG_WITH_LOCATION(Log(), spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), spdlog::level::err, fmtstr##sv, ##__VA_ARGS__) - -#define ZEN_CRITICAL(fmtstr, ...) \ - ZEN_LOG_WITH_LOCATION(Log(), \ - spdlog::source_loc(__FILE__, __LINE__, SPDLOG_FUNCTION), \ - spdlog::level::critical, \ - fmtstr##sv, \ - ##__VA_ARGS__) - -#define ZEN_CONSOLE(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - ConsoleLog().info(fmtstr##sv, ##__VA_ARGS__); \ - } while (false) diff --git a/zencore/include/zencore/md5.h b/zencore/include/zencore/md5.h deleted file mode 100644 index d934dd86b..000000000 --- a/zencore/include/zencore/md5.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <stdint.h> -#include <compare> -#include "zencore.h" - -namespace zen { - -class StringBuilderBase; - -struct MD5 -{ - uint8_t Hash[16]; - - inline auto operator<=>(const MD5& rhs) const = default; - - static const int StringLength = 32; - typedef char String_t[StringLength + 1]; - - static MD5 HashMemory(const void* data, size_t byteCount); - static MD5 FromHexString(const char* string); - const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const; - StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; - - static MD5 Zero; // Initialized to all zeroes -}; - -/** - * Utility class for computing MD5 hashes - */ -class MD5Stream -{ -public: - MD5Stream(); - - /// Begin streaming MD5 compute (not needed on freshly constructed MD5Stream instance) - void Reset(); - /// Append another chunk - MD5Stream& Append(const void* data, size_t byteCount); - /// Obtain final MD5 hash. If you wish to reuse the MD5Stream instance call reset() - MD5 GetHash(); - -private: -}; - -void md5_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h deleted file mode 100644 index 560fa9ffc..000000000 --- a/zencore/include/zencore/memory.h +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <zencore/intmath.h> -#include <zencore/thread.h> - -#include <cstddef> -#include <cstring> -#include <span> -#include <vector> - -namespace zen { - -#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 201911L -template<typename T> -concept ContiguousRange = std::ranges::contiguous_range<T>; -#else -template<typename T> -concept ContiguousRange = true; -#endif - -struct MemoryView; - -class MemoryArena -{ -public: - ZENCORE_API MemoryArena(); - ZENCORE_API ~MemoryArena(); - - ZENCORE_API void* Alloc(size_t Size, size_t Alignment); - ZENCORE_API void Free(void* Ptr); - -private: -}; - -class Memory -{ -public: - ZENCORE_API static void* Alloc(size_t Size, size_t Alignment = sizeof(void*)); - ZENCORE_API static void Free(void* Ptr); -}; - -/** Allocator which claims fixed-size blocks from the underlying allocator. - - There is no way to free individual memory blocks. - - \note This is not thread-safe, you will need to provide synchronization yourself -*/ - -class ChunkingLinearAllocator -{ -public: - ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment = sizeof(std::max_align_t)); - ~ChunkingLinearAllocator(); - - ZENCORE_API void Reset(); - - ZENCORE_API void* Alloc(size_t Size, size_t Alignment = sizeof(void*)); - inline void Free(void* Ptr) { ZEN_UNUSED(Ptr); /* no-op */ } - - ChunkingLinearAllocator(const ChunkingLinearAllocator&) = delete; - ChunkingLinearAllocator& operator=(const ChunkingLinearAllocator&) = delete; - -private: - uint8_t* m_ChunkCursor = nullptr; - uint64_t m_ChunkBytesRemain = 0; - const uint64_t m_ChunkSize = 0; - const uint64_t m_ChunkAlignment = 0; - std::vector<void*> m_ChunkList; -}; - -////////////////////////////////////////////////////////////////////////// - -struct MutableMemoryView -{ - MutableMemoryView() = default; - - MutableMemoryView(void* DataPtr, size_t DataSize) - : m_Data(reinterpret_cast<uint8_t*>(DataPtr)) - , m_DataEnd(reinterpret_cast<uint8_t*>(DataPtr) + DataSize) - { - } - - MutableMemoryView(void* DataPtr, void* DataEndPtr) - : m_Data(reinterpret_cast<uint8_t*>(DataPtr)) - , m_DataEnd(reinterpret_cast<uint8_t*>(DataEndPtr)) - { - } - - inline bool IsEmpty() const { return m_Data == m_DataEnd; } - void* GetData() const { return m_Data; } - void* GetDataEnd() const { return m_DataEnd; } - size_t GetSize() const { return reinterpret_cast<uint8_t*>(m_DataEnd) - reinterpret_cast<uint8_t*>(m_Data); } - - inline bool EqualBytes(const MutableMemoryView& InView) const - { - const size_t Size = GetSize(); - - return Size == InView.GetSize() && (memcmp(m_Data, InView.m_Data, Size) == 0); - } - - /** Modifies the view to be the given number of bytes from the right. */ - inline void RightInline(uint64_t InSize) - { - const uint64_t OldSize = GetSize(); - const uint64_t NewSize = zen::Min(OldSize, InSize); - m_Data = GetDataAtOffsetNoCheck(OldSize - NewSize); - m_DataEnd = m_Data + NewSize; - } - - /** Returns the right-most part of the view by taking the given number of bytes from the right. */ - [[nodiscard]] inline MutableMemoryView Right(uint64_t InSize) const - { - MutableMemoryView View(*this); - View.RightChopInline(InSize); - return View; - } - - /** Modifies the view by chopping the given number of bytes from the left. */ - inline void RightChopInline(uint64_t InSize) - { - const uint64_t Offset = zen::Min(GetSize(), InSize); - m_Data = GetDataAtOffsetNoCheck(Offset); - } - - /** Returns the left-most part of the view by taking the given number of bytes from the left. */ - constexpr inline MutableMemoryView Left(uint64_t InSize) const - { - MutableMemoryView View(*this); - View.LeftInline(InSize); - return View; - } - - /** Modifies the view to be the given number of bytes from the left. */ - constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = zen::Min(m_DataEnd, m_Data + InSize); } - - /** Modifies the view to be the middle part by taking up to the given number of bytes from the given offset. */ - inline void MidInline(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) - { - RightChopInline(InOffset); - LeftInline(InSize); - } - - /** Returns the middle part of the view by taking up to the given number of bytes from the given position. */ - [[nodiscard]] inline MutableMemoryView Mid(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) const - { - MutableMemoryView View(*this); - View.MidInline(InOffset, InSize); - return View; - } - - /** Returns the right-most part of the view by chopping the given number of bytes from the left. */ - [[nodiscard]] inline MutableMemoryView RightChop(uint64_t InSize) const - { - MutableMemoryView View(*this); - View.RightChopInline(InSize); - return View; - } - - inline MutableMemoryView& operator+=(size_t InSize) - { - RightChopInline(InSize); - return *this; - } - - /** Copies bytes from the input view into this view, and returns the remainder of this view. */ - inline MutableMemoryView CopyFrom(MemoryView InView) const; - -private: - uint8_t* m_Data = nullptr; - uint8_t* m_DataEnd = nullptr; - - /** Returns the data pointer advanced by an offset in bytes. */ - inline constexpr uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; } -}; - -////////////////////////////////////////////////////////////////////////// - -struct MemoryView -{ - MemoryView() = default; - - MemoryView(const MutableMemoryView& MutableView) - : m_Data(reinterpret_cast<const uint8_t*>(MutableView.GetData())) - , m_DataEnd(m_Data + MutableView.GetSize()) - { - } - - MemoryView(const void* DataPtr, size_t DataSize) - : m_Data(reinterpret_cast<const uint8_t*>(DataPtr)) - , m_DataEnd(reinterpret_cast<const uint8_t*>(DataPtr) + DataSize) - { - } - - MemoryView(const void* DataPtr, const void* DataEndPtr) - : m_Data(reinterpret_cast<const uint8_t*>(DataPtr)) - , m_DataEnd(reinterpret_cast<const uint8_t*>(DataEndPtr)) - { - } - - inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); } - inline bool IsEmpty() const { return m_Data == m_DataEnd; } - const void* GetData() const { return m_Data; } - const void* GetDataEnd() const { return m_DataEnd; } - size_t GetSize() const { return reinterpret_cast<const uint8_t*>(m_DataEnd) - reinterpret_cast<const uint8_t*>(m_Data); } - inline bool operator==(const MemoryView& Rhs) const { return m_Data == Rhs.m_Data && m_DataEnd == Rhs.m_DataEnd; } - - inline bool EqualBytes(const MemoryView& InView) const - { - const size_t Size = GetSize(); - - return Size == InView.GetSize() && (memcmp(m_Data, InView.GetData(), Size) == 0); - } - - inline MemoryView& operator+=(size_t InSize) - { - RightChopInline(InSize); - return *this; - } - - /** Modifies the view by chopping the given number of bytes from the left. */ - inline void RightChopInline(uint64_t InSize) - { - const uint64_t Offset = zen::Min(GetSize(), InSize); - m_Data = GetDataAtOffsetNoCheck(Offset); - } - - inline MemoryView RightChop(uint64_t InSize) - { - MemoryView View(*this); - View.RightChopInline(InSize); - return View; - } - - /** Returns the right-most part of the view by taking the given number of bytes from the right. */ - [[nodiscard]] inline MemoryView Right(uint64_t InSize) const - { - MemoryView View(*this); - View.RightInline(InSize); - return View; - } - - /** Modifies the view to be the given number of bytes from the right. */ - inline void RightInline(uint64_t InSize) - { - const uint64_t OldSize = GetSize(); - const uint64_t NewSize = zen::Min(OldSize, InSize); - m_Data = GetDataAtOffsetNoCheck(OldSize - NewSize); - m_DataEnd = m_Data + NewSize; - } - - /** Returns the left-most part of the view by taking the given number of bytes from the left. */ - inline MemoryView Left(uint64_t InSize) const - { - MemoryView View(*this); - View.LeftInline(InSize); - return View; - } - - /** Modifies the view to be the given number of bytes from the left. */ - inline void LeftInline(uint64_t InSize) - { - InSize = zen::Min(GetSize(), InSize); - m_DataEnd = zen::Min(m_DataEnd, m_Data + InSize); - } - - /** Modifies the view to be the middle part by taking up to the given number of bytes from the given offset. */ - inline void MidInline(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) - { - RightChopInline(InOffset); - LeftInline(InSize); - } - - /** Returns the middle part of the view by taking up to the given number of bytes from the given position. */ - [[nodiscard]] inline MemoryView Mid(uint64_t InOffset, uint64_t InSize = ~uint64_t(0)) const - { - MemoryView View(*this); - View.MidInline(InOffset, InSize); - return View; - } - - constexpr void Reset() - { - m_Data = nullptr; - m_DataEnd = nullptr; - } - -private: - const uint8_t* m_Data = nullptr; - const uint8_t* m_DataEnd = nullptr; - - /** Returns the data pointer advanced by an offset in bytes. */ - inline constexpr const uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; } -}; - -inline MutableMemoryView -MutableMemoryView::CopyFrom(MemoryView InView) const -{ - ZEN_ASSERT(InView.GetSize() <= GetSize()); - memcpy(m_Data, InView.GetData(), InView.GetSize()); - return RightChop(InView.GetSize()); -} - -/** Advances the start of the view by an offset, which is clamped to stay within the view. */ -inline MemoryView -operator+(const MemoryView& View, uint64_t Offset) -{ - return MemoryView(View) += Offset; -} - -/** Advances the start of the view by an offset, which is clamped to stay within the view. */ -inline MemoryView -operator+(uint64_t Offset, const MemoryView& View) -{ - return MemoryView(View) += Offset; -} - -/** Advances the start of the view by an offset, which is clamped to stay within the view. */ -inline MutableMemoryView -operator+(const MutableMemoryView& View, uint64_t Offset) -{ - return MutableMemoryView(View) += Offset; -} - -/** Advances the start of the view by an offset, which is clamped to stay within the view. */ -inline MutableMemoryView -operator+(uint64_t Offset, const MutableMemoryView& View) -{ - return MutableMemoryView(View) += Offset; -} - -/** - * Make a non-owning view of the memory of the initializer list. - * - * This overload is only available when the element type does not need to be deduced. - */ -template<typename T> -[[nodiscard]] inline MemoryView -MakeMemoryView(std::initializer_list<typename std::type_identity<T>::type> List) -{ - return MemoryView(List.begin(), List.size() * sizeof(T)); -} - -/** Make a non-owning view of the memory of the contiguous container. */ -template<ContiguousRange R> -[[nodiscard]] constexpr inline MemoryView -MakeMemoryView(const R& Container) -{ - std::span Span = Container; - return MemoryView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type)); -} - -/** Make a non-owning const view starting at Data and ending at DataEnd. */ - -[[nodiscard]] inline MemoryView -MakeMemoryView(const void* Data, const void* DataEnd) -{ - return MemoryView(Data, DataEnd); -} - -[[nodiscard]] inline MemoryView -MakeMemoryView(const void* Data, uint64_t Size) -{ - return MemoryView(Data, reinterpret_cast<const uint8_t*>(Data) + Size); -} - -/** - * Make a non-owning mutable view of the memory of the initializer list. - * - * This overload is only available when the element type does not need to be deduced. - */ -template<typename T> -[[nodiscard]] inline MutableMemoryView -MakeMutableMemoryView(std::initializer_list<typename std::type_identity<T>::type> List) -{ - return MutableMemoryView(List.begin(), List.size() * sizeof(T)); -} - -/** Make a non-owning mutable view of the memory of the contiguous container. */ -template<ContiguousRange R> -[[nodiscard]] constexpr inline MutableMemoryView -MakeMutableMemoryView(R& Container) -{ - std::span Span = Container; - return MutableMemoryView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type)); -} - -/** Make a non-owning mutable view starting at Data and ending at DataEnd. */ - -[[nodiscard]] inline MutableMemoryView -MakeMutableMemoryView(void* Data, void* DataEnd) -{ - return MutableMemoryView(Data, DataEnd); -} - -void memory_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/meta.h b/zencore/include/zencore/meta.h deleted file mode 100644 index 82eb5cc30..000000000 --- a/zencore/include/zencore/meta.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -/* This file contains utility functions for meta programming - * - * Since you're in here you're probably quite observant, and you'll - * note that it's quite barren here. This is because template - * metaprogramming is awful and I try not to engage in it. However, - * sometimes these things are forced upon us. - * - */ - -namespace zen { - -/** - * Uses implicit conversion to create an instance of a specific type. - * Useful to make things clearer or circumvent unintended type deduction in templates. - * Safer than C casts and static_casts, e.g. does not allow down-casts - * - * @param Obj The object (usually pointer or reference) to convert. - * - * @return The object converted to the specified type. - */ -template<typename T> -inline T -ImplicitConv(typename std::type_identity<T>::type Obj) -{ - return Obj; -} - -} // namespace zen diff --git a/zencore/include/zencore/mpscqueue.h b/zencore/include/zencore/mpscqueue.h deleted file mode 100644 index 19e410d85..000000000 --- a/zencore/include/zencore/mpscqueue.h +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <atomic> -#include <memory> -#include <new> -#include <optional> - -#ifdef __cpp_lib_hardware_interference_size -using std::hardware_constructive_interference_size; -using std::hardware_destructive_interference_size; -#else -// 64 bytes on x86-64 │ L1_CACHE_BYTES │ L1_CACHE_SHIFT │ __cacheline_aligned │ ... -constexpr std::size_t hardware_constructive_interference_size = 64; -constexpr std::size_t hardware_destructive_interference_size = 64; -#endif - -namespace zen { - -/** An untyped array of data with compile-time alignment and size derived from another type. */ -template<typename ElementType> -struct TypeCompatibleStorage -{ - ElementType* Data() { return (ElementType*)this; } - const ElementType* Data() const { return (const ElementType*)this; } - - alignas(ElementType) char DataMember; -}; - -/** Fast multi-producer/single-consumer unbounded concurrent queue. - - Based on http://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue - */ - -template<typename T> -class MpscQueue final -{ -public: - using ElementType = T; - - MpscQueue() - { - Node* Sentinel = new Node; - Head.store(Sentinel, std::memory_order_relaxed); - Tail = Sentinel; - } - - ~MpscQueue() - { - Node* Next = Tail->Next.load(std::memory_order_relaxed); - - // sentinel's value is already destroyed - delete Tail; - - while (Next != nullptr) - { - Tail = Next; - Next = Tail->Next.load(std::memory_order_relaxed); - - std::destroy_at((ElementType*)&Tail->Value); - delete Tail; - } - } - - template<typename... ArgTypes> - void Enqueue(ArgTypes&&... Args) - { - Node* New = new Node; - new (&New->Value) ElementType(std::forward<ArgTypes>(Args)...); - - Node* Prev = Head.exchange(New, std::memory_order_acq_rel); - Prev->Next.store(New, std::memory_order_release); - } - - std::optional<ElementType> Dequeue() - { - Node* Next = Tail->Next.load(std::memory_order_acquire); - - if (Next == nullptr) - { - return {}; - } - - ElementType* ValuePtr = (ElementType*)&Next->Value; - std::optional<ElementType> Res{std::move(*ValuePtr)}; - std::destroy_at(ValuePtr); - - delete Tail; // current sentinel - - Tail = Next; // new sentinel - return Res; - } - -private: - struct Node - { - std::atomic<Node*> Next{nullptr}; - TypeCompatibleStorage<ElementType> Value; - }; - -private: - std::atomic<Node*> Head; // accessed only by producers - alignas(hardware_constructive_interference_size) - Node* Tail; // accessed only by consumer, hence should be on a different cache line than `Head` -}; - -void mpscqueue_forcelink(); - -} // namespace zen diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h deleted file mode 100644 index f0bb6b85e..000000000 --- a/zencore/include/zencore/refcount.h +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. -#pragma once - -#include "atomic.h" -#include "zencore.h" - -#include <compare> - -namespace zen { - -/** - * Helper base class for reference counted objects using intrusive reference counts - * - * This class is pretty straightforward but does one thing which may be unexpected: - * - * - Instances on the stack are initialized with a reference count of one to ensure - * nobody tries to accidentally delete it. (TODO: is this really useful?) - */ -class RefCounted -{ -public: - RefCounted() = default; - virtual ~RefCounted() = default; - - inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); } - inline uint32_t Release() const - { - uint32_t refCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount); - if (refCount == 0) - { - delete this; - } - return refCount; - } - - // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it - - RefCounted(const RefCounted&) = delete; - RefCounted(RefCounted&&) = delete; - RefCounted& operator=(const RefCounted&) = delete; - RefCounted& operator=(RefCounted&&) = delete; - -protected: - inline uint32_t RefCount() const { return m_RefCount; } - -private: - uint32_t m_RefCount = 0; -}; - -/** - * Smart pointer for classes derived from RefCounted - */ - -template<class T> -class RefPtr -{ -public: - inline RefPtr() = default; - inline RefPtr(const RefPtr& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } - inline RefPtr(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } - inline ~RefPtr() { m_Ref && m_Ref->Release(); } - - [[nodiscard]] inline bool IsNull() const { return m_Ref == nullptr; } - inline explicit operator bool() const { return m_Ref != nullptr; } - inline operator T*() const { return m_Ref; } - inline T* operator->() const { return m_Ref; } - - inline std::strong_ordering operator<=>(const RefPtr& Rhs) const = default; - - inline RefPtr& operator=(T* Rhs) - { - Rhs && Rhs->AddRef(); - m_Ref && m_Ref->Release(); - m_Ref = Rhs; - return *this; - } - inline RefPtr& operator=(const RefPtr& Rhs) - { - if (&Rhs != this) - { - Rhs && Rhs->AddRef(); - m_Ref && m_Ref->Release(); - m_Ref = Rhs.m_Ref; - } - return *this; - } - inline RefPtr& operator=(RefPtr&& Rhs) noexcept - { - if (&Rhs != this) - { - m_Ref && m_Ref->Release(); - m_Ref = Rhs.m_Ref; - Rhs.m_Ref = nullptr; - } - return *this; - } - template<typename OtherType> - inline RefPtr& operator=(RefPtr<OtherType>&& Rhs) noexcept - { - if ((RefPtr*)&Rhs != this) - { - m_Ref && m_Ref->Release(); - m_Ref = Rhs.m_Ref; - Rhs.m_Ref = nullptr; - } - return *this; - } - inline RefPtr(RefPtr&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } - template<typename OtherType> - explicit inline RefPtr(RefPtr<OtherType>&& Rhs) noexcept : m_Ref(Rhs.m_Ref) - { - Rhs.m_Ref = nullptr; - } - -private: - T* m_Ref = nullptr; - template<typename U> - friend class RefPtr; -}; - -/** - * Smart pointer for classes derived from RefCounted - * - * This variant does not decay to a raw pointer - * - */ - -template<class T> -class Ref -{ -public: - inline Ref() = default; - inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } - inline explicit Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } - inline ~Ref() { m_Ref && m_Ref->Release(); } - - template<typename DerivedType> - requires DerivedFrom<DerivedType, T> - inline Ref(const Ref<DerivedType>& Rhs) : Ref(Rhs.m_Ref) {} - - [[nodiscard]] inline bool IsNull() const { return m_Ref == nullptr; } - inline explicit operator bool() const { return m_Ref != nullptr; } - inline T* operator->() const { return m_Ref; } - inline T* Get() const { return m_Ref; } - - inline std::strong_ordering operator<=>(const Ref& Rhs) const = default; - - inline Ref& operator=(T* Rhs) - { - Rhs && Rhs->AddRef(); - m_Ref && m_Ref->Release(); - m_Ref = Rhs; - return *this; - } - inline Ref& operator=(const Ref& Rhs) - { - if (&Rhs != this) - { - Rhs && Rhs->AddRef(); - m_Ref && m_Ref->Release(); - m_Ref = Rhs.m_Ref; - } - return *this; - } - inline Ref& operator=(Ref&& Rhs) noexcept - { - if (&Rhs != this) - { - m_Ref && m_Ref->Release(); - m_Ref = Rhs.m_Ref; - Rhs.m_Ref = nullptr; - } - return *this; - } - inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } - -private: - T* m_Ref = nullptr; - - template<class U> - friend class Ref; -}; - -void refcount_forcelink(); - -} // namespace zen diff --git a/zencore/include/zencore/scopeguard.h b/zencore/include/zencore/scopeguard.h deleted file mode 100644 index d04c8ed9c..000000000 --- a/zencore/include/zencore/scopeguard.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <type_traits> -#include "logging.h" -#include "zencore.h" - -namespace zen { - -template<typename T> -class [[nodiscard]] ScopeGuardImpl -{ -public: - inline ScopeGuardImpl(T&& func) : m_guardFunc(func) {} - ~ScopeGuardImpl() - { - if (!m_dismissed) - { - try - { - m_guardFunc(); - } - catch (std::exception& Ex) - { - ZEN_ERROR("scope guard threw exception: '{}'", Ex.what()); - } - } - } - - void Dismiss() { m_dismissed = true; } - -private: - bool m_dismissed = false; - T m_guardFunc; -}; - -template<typename T> -ScopeGuardImpl<T> -MakeGuard(T&& fn) -{ - return ScopeGuardImpl<T>(std::move(fn)); -} - -} // namespace zen diff --git a/zencore/include/zencore/session.h b/zencore/include/zencore/session.h deleted file mode 100644 index dd90197bf..000000000 --- a/zencore/include/zencore/session.h +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -namespace zen { - -struct Oid; - -ZENCORE_API [[nodiscard]] Oid GetSessionId(); -ZENCORE_API [[nodiscard]] std::string_view GetSessionIdString(); - -} // namespace zen diff --git a/zencore/include/zencore/sha1.h b/zencore/include/zencore/sha1.h deleted file mode 100644 index fc26f442b..000000000 --- a/zencore/include/zencore/sha1.h +++ /dev/null @@ -1,76 +0,0 @@ -// ////////////////////////////////////////////////////////// -// sha1.h -// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. -// see http://create.stephan-brumme.com/disclaimer.html -// - -#pragma once - -#include <stdint.h> -#include <compare> -#include "zencore.h" - -namespace zen { - -class StringBuilderBase; - -struct SHA1 -{ - uint8_t Hash[20]; - - inline auto operator<=>(const SHA1& rhs) const = default; - - static const int StringLength = 40; - typedef char String_t[StringLength + 1]; - - static SHA1 HashMemory(const void* data, size_t byteCount); - static SHA1 FromHexString(const char* string); - const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const; - StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; - - static SHA1 Zero; // Initialized to all zeroes -}; - -/** - * Utility class for computing SHA1 hashes - */ -class SHA1Stream -{ -public: - SHA1Stream(); - - /** compute SHA1 of a memory block - - \note SHA1 class contains a slightly more convenient helper function for this use case - \see SHA1::fromMemory() - */ - SHA1 Compute(const void* data, size_t byteCount); - - /// Begin streaming SHA1 compute (not needed on freshly constructed SHA1Stream instance) - void Reset(); - /// Append another chunk - SHA1Stream& Append(const void* data, size_t byteCount); - /// Obtain final SHA1 hash. If you wish to reuse the SHA1Stream instance call reset() - SHA1 GetHash(); - -private: - void ProcessBlock(const void* data); - void ProcessBuffer(); - - enum - { - /// split into 64 byte blocks (=> 512 bits) - BlockSize = 512 / 8, - HashBytes = 20, - HashValues = HashBytes / 4 - }; - - uint64_t m_NumBytes; // size of processed data in bytes - size_t m_BufferSize; // valid bytes in m_buffer - uint8_t m_Buffer[BlockSize]; // bytes not processed yet - uint32_t m_Hash[HashValues]; -}; - -void sha1_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h deleted file mode 100644 index 97c5a9d21..000000000 --- a/zencore/include/zencore/sharedbuffer.h +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <zencore/iobuffer.h> -#include <zencore/memory.h> -#include <zencore/refcount.h> - -#include <memory.h> - -namespace zen { - -class SharedBuffer; - -/** - * Reference to a memory buffer with a single owner - * - * Internally - */ -class UniqueBuffer -{ -public: - UniqueBuffer() = default; - UniqueBuffer(UniqueBuffer&&) = default; - UniqueBuffer& operator=(UniqueBuffer&&) = default; - UniqueBuffer(const UniqueBuffer&) = delete; - UniqueBuffer& operator=(const UniqueBuffer&) = delete; - - ZENCORE_API explicit UniqueBuffer(IoBufferCore* Owner); - - [[nodiscard]] void* GetData() { return m_Buffer ? m_Buffer->MutableDataPointer() : nullptr; } - [[nodiscard]] const void* GetData() const { return m_Buffer ? m_Buffer->DataPointer() : nullptr; } - [[nodiscard]] size_t GetSize() const { return m_Buffer ? m_Buffer->DataBytes() : 0; } - - operator MutableMemoryView() { return GetMutableView(); } - operator MemoryView() const { return GetView(); } - - /** - * Returns true if this does not point to a buffer owner. - * - * A null buffer is always owned, materialized, and empty. - */ - [[nodiscard]] inline bool IsNull() const { return m_Buffer.IsNull(); } - - /** Reset this to null. */ - ZENCORE_API void Reset(); - - [[nodiscard]] inline MutableMemoryView GetMutableView() { return MutableMemoryView(GetData(), GetSize()); } - [[nodiscard]] inline MemoryView GetView() const { return MemoryView(GetData(), GetSize()); } - - /** Make an uninitialized owned buffer of the specified size. */ - [[nodiscard]] ZENCORE_API static UniqueBuffer Alloc(uint64_t Size); - - /** Make a non-owned view of the input. */ - [[nodiscard]] ZENCORE_API static UniqueBuffer MakeMutableView(void* DataPtr, uint64_t Size); - - /** - * Convert this to an immutable shared buffer, leaving this null. - * - * Steals the buffer owner from the unique buffer. - */ - [[nodiscard]] ZENCORE_API SharedBuffer MoveToShared(); - -private: - // This may be null, for a default constructed UniqueBuffer only - RefPtr<IoBufferCore> m_Buffer; - - friend class SharedBuffer; -}; - -/** - * Reference to a memory buffer with shared ownership - */ -class SharedBuffer -{ -public: - SharedBuffer() = default; - ZENCORE_API explicit SharedBuffer(UniqueBuffer&&); - inline explicit SharedBuffer(IoBufferCore* Owner) : m_Buffer(Owner) {} - ZENCORE_API explicit SharedBuffer(IoBuffer&& Buffer) : m_Buffer(std::move(Buffer.m_Core)) {} - ZENCORE_API explicit SharedBuffer(const IoBuffer& Buffer) : m_Buffer(Buffer.m_Core) {} - ZENCORE_API explicit SharedBuffer(RefPtr<IoBufferCore>&& Owner) : m_Buffer(std::move(Owner)) {} - - [[nodiscard]] const void* GetData() const - { - if (m_Buffer) - { - return m_Buffer->DataPointer(); - } - return nullptr; - } - - [[nodiscard]] size_t GetSize() const - { - if (m_Buffer) - { - return m_Buffer->DataBytes(); - } - return 0; - } - - inline void MakeImmutable() - { - ZEN_ASSERT(m_Buffer); - m_Buffer->SetIsImmutable(true); - } - - /** Returns a buffer that is owned, by cloning if not owned. */ - [[nodiscard]] ZENCORE_API SharedBuffer MakeOwned() const&; - [[nodiscard]] ZENCORE_API SharedBuffer MakeOwned() &&; - - [[nodiscard]] bool IsOwned() const { return !m_Buffer || m_Buffer->IsOwned(); } - [[nodiscard]] inline bool IsNull() const { return !m_Buffer; } - inline void Reset() { m_Buffer = nullptr; } - - [[nodiscard]] MemoryView GetView() const - { - if (m_Buffer) - { - return MemoryView(m_Buffer->DataPointer(), m_Buffer->DataBytes()); - } - else - { - return MemoryView(); - } - } - operator MemoryView() const { return GetView(); } - - /** Returns true if this points to a buffer owner. */ - [[nodiscard]] inline explicit operator bool() const { return !IsNull(); } - - [[nodiscard]] inline IoBuffer AsIoBuffer() const { return IoBuffer(m_Buffer); } - - SharedBuffer& operator=(UniqueBuffer&& Rhs) - { - m_Buffer = std::move(Rhs.m_Buffer); - return *this; - } - - std::strong_ordering operator<=>(const SharedBuffer& Rhs) const = default; - - /** Make a non-owned view of the input */ - [[nodiscard]] inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); } - /** Make a non-owning view of the memory of the contiguous container. */ - [[nodiscard]] inline static SharedBuffer MakeView(const ContiguousRange auto& Container) - { - std::span Span = Container; - return MakeView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type)); - } - /** Make a non-owned view of the input */ - [[nodiscard]] ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size); - /** Make a non-owned view of the input */ - [[nodiscard]] ZENCORE_API static SharedBuffer MakeView(MemoryView View, SharedBuffer OuterBuffer); - /** Make an owned clone of the buffer */ - [[nodiscard]] ZENCORE_API SharedBuffer Clone(); - /** Make an owned clone of the memory in the input view */ - [[nodiscard]] ZENCORE_API static SharedBuffer Clone(MemoryView View); - -private: - RefPtr<IoBufferCore> m_Buffer; -}; - -void sharedbuffer_forcelink(); - -} // namespace zen diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h deleted file mode 100644 index 1a0817b99..000000000 --- a/zencore/include/zencore/stats.h +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <atomic> -#include <random> - -namespace zen { -class CbObjectWriter; -} - -namespace zen::metrics { - -template<typename T> -class Gauge -{ -public: - Gauge() : m_Value{0} {} - - T Value() const { return m_Value; } - void SetValue(T Value) { m_Value = Value; } - -private: - std::atomic<T> m_Value; -}; - -/** Stats counter - * - * A counter is modified by adding or subtracting a value from a current value. - * This would typically be used to track number of requests in flight, number - * of active jobs etc - * - */ -class Counter -{ -public: - inline void SetValue(uint64_t Value) { m_count = Value; } - inline uint64_t Value() const { return m_count; } - - inline void Increment(int64_t AddValue) { m_count.fetch_add(AddValue); } - inline void Decrement(int64_t SubValue) { m_count.fetch_sub(SubValue); } - inline void Clear() { m_count.store(0, std::memory_order_release); } - -private: - std::atomic<uint64_t> m_count{0}; -}; - -/** Exponential Weighted Moving Average - - This is very raw, to use as little state as possible. If we - want to use this more broadly in user code we should perhaps - add a more user-friendly wrapper - */ - -class RawEWMA -{ -public: - /// <summary> - /// Update EWMA with new measure - /// </summary> - /// <param name="Alpha">Smoothing factor (between 0 and 1)</param> - /// <param name="Interval">Elapsed time since last</param> - /// <param name="Count">Value</param> - /// <param name="IsInitialUpdate">Whether this is the first update or not</param> - void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate); - double Rate() const; - -private: - std::atomic<double> m_Rate = 0; -}; - -/// <summary> -/// Tracks rate of events over time (i.e requests/sec), using -/// exponential moving averages -/// </summary> -class Meter -{ -public: - Meter(); - ~Meter(); - - inline uint64_t Count() const { return m_TotalCount; } - double Rate1(); // One-minute rate - double Rate5(); // Five-minute rate - double Rate15(); // Fifteen-minute rate - double MeanRate() const; // Mean rate since instantiation of this meter - void Mark(uint64_t Count = 1); // Register one or more events - -private: - std::atomic<uint64_t> m_TotalCount{0}; // Accumulator counting number of marks since beginning - std::atomic<uint64_t> m_PendingCount{0}; // Pending EWMA update accumulator - std::atomic<uint64_t> m_StartTick{0}; // Time this was instantiated (for mean) - std::atomic<uint64_t> m_LastTick{0}; // Timestamp of last EWMA tick - std::atomic<int64_t> m_Remainder{0}; // Tracks the "modulo" of tick time - bool m_IsFirstTick = true; - RawEWMA m_RateM1; - RawEWMA m_RateM5; - RawEWMA m_RateM15; - - void TickIfNecessary(); - void Tick(); -}; - -/** Moment-in-time snapshot of a distribution - */ -class SampleSnapshot -{ -public: - SampleSnapshot(std::vector<double>&& Values); - ~SampleSnapshot(); - - uint32_t Size() const { return (uint32_t)m_Values.size(); } - double GetQuantileValue(double Quantile); - double GetMedian() { return GetQuantileValue(0.5); } - double Get75Percentile() { return GetQuantileValue(0.75); } - double Get95Percentile() { return GetQuantileValue(0.95); } - double Get98Percentile() { return GetQuantileValue(0.98); } - double Get99Percentile() { return GetQuantileValue(0.99); } - double Get999Percentile() { return GetQuantileValue(0.999); } - const std::vector<double>& GetValues() const; - -private: - std::vector<double> m_Values; -}; - -/** Randomly selects samples from a stream. Uses Vitter's - Algorithm R to produce a statistically representative sample. - - http://www.cs.umd.edu/~samir/498/vitter.pdf - Random Sampling with a Reservoir - */ - -class UniformSample -{ -public: - UniformSample(uint32_t ReservoirSize); - ~UniformSample(); - - void Clear(); - uint32_t Size() const; - void Update(int64_t Value); - SampleSnapshot Snapshot() const; - - template<Invocable<int64_t> T> - void IterateValues(T Callback) const - { - for (const auto& Value : m_Values) - { - Callback(Value); - } - } - -private: - std::atomic<uint64_t> m_SampleCounter{0}; - std::vector<std::atomic<int64_t>> m_Values; -}; - -/** Track (probabilistic) sample distribution along with min/max - */ -class Histogram -{ -public: - Histogram(int32_t SampleCount = 1028); - ~Histogram(); - - void Clear(); - void Update(int64_t Value); - int64_t Max() const; - int64_t Min() const; - double Mean() const; - uint64_t Count() const; - SampleSnapshot Snapshot() const { return m_Sample.Snapshot(); } - -private: - UniformSample m_Sample; - std::atomic<int64_t> m_Min{0}; - std::atomic<int64_t> m_Max{0}; - std::atomic<int64_t> m_Sum{0}; - std::atomic<int64_t> m_Count{0}; -}; - -/** Track timing and frequency of some operation - - Example usage would be to track frequency and duration of network - requests, or function calls. - - */ -class OperationTiming -{ -public: - OperationTiming(int32_t SampleCount = 514); - ~OperationTiming(); - - void Update(int64_t Duration); - int64_t Max() const; - int64_t Min() const; - double Mean() const; - uint64_t Count() const; - SampleSnapshot Snapshot() const { return m_Histogram.Snapshot(); } - - double Rate1() { return m_Meter.Rate1(); } - double Rate5() { return m_Meter.Rate5(); } - double Rate15() { return m_Meter.Rate15(); } - double MeanRate() const { return m_Meter.MeanRate(); } - - struct Scope - { - Scope(OperationTiming& Outer); - ~Scope(); - - void Stop(); - void Cancel(); - - private: - OperationTiming& m_Outer; - uint64_t m_StartTick; - }; - -private: - Meter m_Meter; - Histogram m_Histogram; -}; - -/** Metrics for network requests - - Aggregates tracking of duration, payload sizes into a single - class - - */ -class RequestStats -{ -public: - RequestStats(int32_t SampleCount = 514); - ~RequestStats(); - - void Update(int64_t Duration, int64_t Bytes); - uint64_t Count() const; - - // Timing - - int64_t MaxDuration() const { return m_BytesHistogram.Max(); } - int64_t MinDuration() const { return m_BytesHistogram.Min(); } - double MeanDuration() const { return m_BytesHistogram.Mean(); } - SampleSnapshot DurationSnapshot() const { return m_RequestTimeHistogram.Snapshot(); } - double Rate1() { return m_RequestMeter.Rate1(); } - double Rate5() { return m_RequestMeter.Rate5(); } - double Rate15() { return m_RequestMeter.Rate15(); } - double MeanRate() const { return m_RequestMeter.MeanRate(); } - - // Bytes - - int64_t MaxBytes() const { return m_BytesHistogram.Max(); } - int64_t MinBytes() const { return m_BytesHistogram.Min(); } - double MeanBytes() const { return m_BytesHistogram.Mean(); } - SampleSnapshot BytesSnapshot() const { return m_BytesHistogram.Snapshot(); } - double ByteRate1() { return m_BytesMeter.Rate1(); } - double ByteRate5() { return m_BytesMeter.Rate5(); } - double ByteRate15() { return m_BytesMeter.Rate15(); } - double ByteMeanRate() const { return m_BytesMeter.MeanRate(); } - - struct Scope - { - Scope(OperationTiming& Outer); - ~Scope(); - - void Cancel(); - - private: - OperationTiming& m_Outer; - uint64_t m_StartTick; - }; - - void EmitSnapshot(std::string_view Tag, CbObjectWriter& Cbo); - -private: - Meter m_RequestMeter; - Meter m_BytesMeter; - Histogram m_RequestTimeHistogram; - Histogram m_BytesHistogram; -}; - -void EmitSnapshot(std::string_view Tag, OperationTiming& Stat, CbObjectWriter& Cbo); -void EmitSnapshot(std::string_view Tag, const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor); -void EmitSnapshot(std::string_view Tag, Meter& Stat, CbObjectWriter& Cbo); - -void EmitSnapshot(const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor); - -} // namespace zen::metrics - -namespace zen { - -extern void stats_forcelink(); - -} // namespace zen diff --git a/zencore/include/zencore/stream.h b/zencore/include/zencore/stream.h deleted file mode 100644 index 9e4996249..000000000 --- a/zencore/include/zencore/stream.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <zencore/memory.h> -#include <zencore/thread.h> - -#include <vector> - -namespace zen { - -/** - * Binary stream writer - */ - -class BinaryWriter -{ -public: - inline BinaryWriter() = default; - ~BinaryWriter() = default; - - inline void Write(const void* DataPtr, size_t ByteCount) - { - Write(DataPtr, ByteCount, m_Offset); - m_Offset += ByteCount; - } - - inline void Write(MemoryView Memory) { Write(Memory.GetData(), Memory.GetSize()); } - void Write(std::initializer_list<const MemoryView> Buffers); - - inline uint64_t CurrentOffset() const { return m_Offset; } - - inline const uint8_t* Data() const { return m_Buffer.data(); } - inline const uint8_t* GetData() const { return m_Buffer.data(); } - inline uint64_t Size() const { return m_Buffer.size(); } - inline uint64_t GetSize() const { return m_Buffer.size(); } - void Reset(); - - inline MemoryView GetView() const { return MemoryView(m_Buffer.data(), m_Offset); } - inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Buffer.data(), m_Offset); } - -private: - std::vector<uint8_t> m_Buffer; - uint64_t m_Offset = 0; - - void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset); -}; - -inline MemoryView -MakeMemoryView(const BinaryWriter& Stream) -{ - return MemoryView(Stream.Data(), Stream.Size()); -} - -/** - * Binary stream reader - */ - -class BinaryReader -{ -public: - inline BinaryReader(const void* Buffer, uint64_t Size) : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer)), m_BufferSize(Size) {} - inline BinaryReader(MemoryView Buffer) - : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer.GetData())) - , m_BufferSize(Buffer.GetSize()) - { - } - - inline void Read(void* DataPtr, size_t ByteCount) - { - memcpy(DataPtr, m_BufferBase + m_Offset, ByteCount); - m_Offset += ByteCount; - } - - inline uint64_t Size() const { return m_BufferSize; } - inline uint64_t GetSize() const { return Size(); } - inline uint64_t CurrentOffset() const { return m_Offset; } - inline void Skip(size_t ByteCount) { m_Offset += ByteCount; }; - -private: - const uint8_t* m_BufferBase; - uint64_t m_BufferSize; - uint64_t m_Offset = 0; -}; - -void stream_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h deleted file mode 100644 index ab111ff81..000000000 --- a/zencore/include/zencore/string.h +++ /dev/null @@ -1,1115 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "intmath.h" -#include "zencore.h" - -#include <stdint.h> -#include <string.h> -#include <charconv> -#include <codecvt> -#include <compare> -#include <concepts> -#include <optional> -#include <span> -#include <string_view> - -#include <type_traits> - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -inline bool -StringEquals(const char8_t* s1, const char* s2) -{ - return strcmp(reinterpret_cast<const char*>(s1), s2) == 0; -} - -inline bool -StringEquals(const char* s1, const char* s2) -{ - return strcmp(s1, s2) == 0; -} - -inline size_t -StringLength(const char* str) -{ - return strlen(str); -} - -inline bool -StringEquals(const wchar_t* s1, const wchar_t* s2) -{ - return wcscmp(s1, s2) == 0; -} - -inline size_t -StringLength(const wchar_t* str) -{ - return wcslen(str); -} - -////////////////////////////////////////////////////////////////////////// -// File name helpers -// - -ZENCORE_API const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr); - -////////////////////////////////////////////////////////////////////////// -// Text formatting of numbers -// - -ZENCORE_API bool ToString(std::span<char> Buffer, uint64_t Num); -ZENCORE_API bool ToString(std::span<char> Buffer, int64_t Num); - -struct TextNumBase -{ - inline const char* c_str() const { return m_Buffer; } - inline operator std::string_view() const { return std::string_view(m_Buffer); } - -protected: - char m_Buffer[24]; -}; - -struct IntNum : public TextNumBase -{ - inline IntNum(UnsignedIntegral auto Number) { ToString(m_Buffer, uint64_t(Number)); } - inline IntNum(SignedIntegral auto Number) { ToString(m_Buffer, int64_t(Number)); } -}; - -////////////////////////////////////////////////////////////////////////// -// -// Quick-and-dirty string builder. Good enough for me, but contains traps -// and not-quite-ideal behaviour especially when mixing character types etc -// - -template<typename C> -class StringBuilderImpl -{ -public: - StringBuilderImpl() = default; - ZENCORE_API ~StringBuilderImpl(); - - StringBuilderImpl(const StringBuilderImpl&) = delete; - StringBuilderImpl(const StringBuilderImpl&&) = delete; - const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete; - const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete; - - inline size_t AddUninitialized(size_t Count) - { - EnsureCapacity(Count); - const size_t OldCount = Size(); - m_CurPos += Count; - return OldCount; - } - - StringBuilderImpl& Append(C OneChar) - { - EnsureCapacity(1); - - *m_CurPos++ = OneChar; - - return *this; - } - - inline StringBuilderImpl& AppendAscii(const std::string_view& String) - { - const size_t len = String.size(); - - EnsureCapacity(len); - - for (size_t i = 0; i < len; ++i) - m_CurPos[i] = String[i]; - - m_CurPos += len; - - return *this; - } - - inline StringBuilderImpl& AppendAscii(const std::u8string_view& String) - { - const size_t len = String.size(); - - EnsureCapacity(len); - - for (size_t i = 0; i < len; ++i) - m_CurPos[i] = String[i]; - - m_CurPos += len; - - return *this; - } - - inline StringBuilderImpl& AppendAscii(const char* NulTerminatedString) - { - size_t StringLen = StringLength(NulTerminatedString); - - return AppendAscii({NulTerminatedString, StringLen}); - } - - inline StringBuilderImpl& Append(const char8_t* NulTerminatedString) - { - // This is super hacky and not fully functional - needs better - // solution - if constexpr (sizeof(C) == 1) - { - size_t len = StringLength((const char*)NulTerminatedString); - - EnsureCapacity(len); - - for (size_t i = 0; i < len; ++i) - m_CurPos[i] = C(NulTerminatedString[i]); - - m_CurPos += len; - } - else - { - ZEN_NOT_IMPLEMENTED(); - } - - return *this; - } - - inline StringBuilderImpl& AppendAsciiRange(const char* BeginString, const char* EndString) - { - EnsureCapacity(EndString - BeginString); - - while (BeginString != EndString) - *m_CurPos++ = *BeginString++; - - return *this; - } - - inline StringBuilderImpl& Append(const C* NulTerminatedString) - { - size_t Len = StringLength(NulTerminatedString); - - EnsureCapacity(Len); - memcpy(m_CurPos, NulTerminatedString, Len * sizeof(C)); - m_CurPos += Len; - - return *this; - } - - inline StringBuilderImpl& Append(const C* NulTerminatedString, size_t MaxChars) - { - size_t len = Min(MaxChars, StringLength(NulTerminatedString)); - - EnsureCapacity(len); - memcpy(m_CurPos, NulTerminatedString, len * sizeof(C)); - m_CurPos += len; - - return *this; - } - - inline StringBuilderImpl& AppendRange(const C* BeginString, const C* EndString) - { - size_t Len = EndString - BeginString; - - EnsureCapacity(Len); - memcpy(m_CurPos, BeginString, Len * sizeof(C)); - m_CurPos += Len; - - return *this; - } - - inline StringBuilderImpl& Append(const std::basic_string_view<C>& String) - { - return AppendRange(String.data(), String.data() + String.size()); - } - - inline StringBuilderImpl& AppendBool(bool v) - { - // This is a method instead of a << operator overload as the latter can - // easily get called with non-bool types like pointers. It is a very - // subtle behaviour that can cause bugs. - using namespace std::literals; - if (v) - { - return AppendAscii("true"sv); - } - return AppendAscii("false"sv); - } - - inline void RemoveSuffix(uint32_t Count) - { - ZEN_ASSERT(Count <= Size()); - m_CurPos -= Count; - } - - inline const C* c_str() const - { - EnsureNulTerminated(); - return m_Base; - } - - inline C* Data() - { - EnsureNulTerminated(); - return m_Base; - } - - inline const C* Data() const - { - EnsureNulTerminated(); - return m_Base; - } - - inline size_t Size() const { return m_CurPos - m_Base; } - inline bool IsDynamic() const { return m_IsDynamic; } - inline void Reset() { m_CurPos = m_Base; } - - inline StringBuilderImpl& operator<<(uint64_t n) - { - IntNum Str(n); - return AppendAscii(Str); - } - inline StringBuilderImpl& operator<<(int64_t n) - { - IntNum Str(n); - return AppendAscii(Str); - } - inline StringBuilderImpl& operator<<(uint32_t n) - { - IntNum Str(n); - return AppendAscii(Str); - } - inline StringBuilderImpl& operator<<(int32_t n) - { - IntNum Str(n); - return AppendAscii(Str); - } - inline StringBuilderImpl& operator<<(uint16_t n) - { - IntNum Str(n); - return AppendAscii(Str); - } - inline StringBuilderImpl& operator<<(int16_t n) - { - IntNum Str(n); - return AppendAscii(Str); - } - inline StringBuilderImpl& operator<<(uint8_t n) - { - IntNum Str(n); - return AppendAscii(Str); - } - inline StringBuilderImpl& operator<<(int8_t n) - { - IntNum Str(n); - return AppendAscii(Str); - } - - inline StringBuilderImpl& operator<<(const char* str) { return AppendAscii(str); } - inline StringBuilderImpl& operator<<(const std::string_view str) { return AppendAscii(str); } - inline StringBuilderImpl& operator<<(const std::u8string_view str) { return AppendAscii(str); } - -protected: - inline void Init(C* Base, size_t Capacity) - { - m_Base = m_CurPos = Base; - m_End = Base + Capacity; - } - - inline void EnsureNulTerminated() const { *m_CurPos = '\0'; } - - inline void EnsureCapacity(size_t ExtraRequired) - { - // precondition: we know the current buffer has enough capacity - // for the existing string including NUL terminator - - if ((m_CurPos + ExtraRequired) < m_End) - return; - - Extend(ExtraRequired); - } - - ZENCORE_API void Extend(size_t ExtraCapacity); - ZENCORE_API void* AllocBuffer(size_t ByteCount); - ZENCORE_API void FreeBuffer(void* Buffer, size_t ByteCount); - - ZENCORE_API [[noreturn]] void Fail(const char* FailReason); // note: throws exception - - C* m_Base; - C* m_CurPos; - C* m_End; - bool m_IsDynamic = false; - bool m_IsExtendable = false; -}; - -////////////////////////////////////////////////////////////////////////// - -extern template class StringBuilderImpl<char>; - -inline StringBuilderImpl<char>& -operator<<(StringBuilderImpl<char>& Builder, char Char) -{ - return Builder.Append(Char); -} - -class StringBuilderBase : public StringBuilderImpl<char> -{ -public: - inline StringBuilderBase(char* bufferPointer, size_t bufferCapacity) { Init(bufferPointer, bufferCapacity); } - inline ~StringBuilderBase() = default; - - // Note that we don't need a terminator for the string_view so we avoid calling data() here - inline operator std::string_view() const { return std::string_view(m_Base, m_CurPos - m_Base); } - inline std::string_view ToView() const { return std::string_view(m_Base, m_CurPos - m_Base); } - inline std::string ToString() const { return std::string{Data(), Size()}; } - - inline void AppendCodepoint(uint32_t cp) - { - if (cp < 0x80) // one octet - { - Append(static_cast<char8_t>(cp)); - } - else if (cp < 0x800) - { - EnsureCapacity(2); // two octets - m_CurPos[0] = static_cast<char8_t>((cp >> 6) | 0xc0); - m_CurPos[1] = static_cast<char8_t>((cp & 0x3f) | 0x80); - m_CurPos += 2; - } - else if (cp < 0x10000) - { - EnsureCapacity(3); // three octets - m_CurPos[0] = static_cast<char8_t>((cp >> 12) | 0xe0); - m_CurPos[1] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80); - m_CurPos[2] = static_cast<char8_t>((cp & 0x3f) | 0x80); - m_CurPos += 3; - } - else - { - EnsureCapacity(4); // four octets - m_CurPos[0] = static_cast<char8_t>((cp >> 18) | 0xf0); - m_CurPos[1] = static_cast<char8_t>(((cp >> 12) & 0x3f) | 0x80); - m_CurPos[2] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80); - m_CurPos[3] = static_cast<char8_t>((cp & 0x3f) | 0x80); - m_CurPos += 4; - } - } -}; - -template<size_t N> -class StringBuilder : public StringBuilderBase -{ -public: - inline StringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) {} - inline ~StringBuilder() = default; - -private: - char m_StringBuffer[N]; -}; - -template<size_t N> -class ExtendableStringBuilder : public StringBuilderBase -{ -public: - inline ExtendableStringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) { m_IsExtendable = true; } - inline ~ExtendableStringBuilder() = default; - -private: - char m_StringBuffer[N]; -}; - -template<size_t N> -class WriteToString : public ExtendableStringBuilder<N> -{ -public: - template<typename... ArgTypes> - explicit WriteToString(ArgTypes&&... Args) - { - (*this << ... << std::forward<ArgTypes>(Args)); - } -}; - -////////////////////////////////////////////////////////////////////////// - -extern template class StringBuilderImpl<wchar_t>; - -class WideStringBuilderBase : public StringBuilderImpl<wchar_t> -{ -public: - inline WideStringBuilderBase(wchar_t* BufferPointer, size_t BufferCapacity) { Init(BufferPointer, BufferCapacity); } - inline ~WideStringBuilderBase() = default; - - inline operator std::wstring_view() const { return std::wstring_view{Data(), Size()}; } - inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; } - inline std::wstring ToString() const { return std::wstring{Data(), Size()}; } - - inline StringBuilderImpl& operator<<(const std::wstring_view str) { return Append((const wchar_t*)str.data(), str.size()); } - inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); } - using StringBuilderImpl:: operator<<; -}; - -template<size_t N> -class WideStringBuilder : public WideStringBuilderBase -{ -public: - inline WideStringBuilder() : WideStringBuilderBase(m_Buffer, N) {} - ~WideStringBuilder() = default; - -private: - wchar_t m_Buffer[N]; -}; - -template<size_t N> -class ExtendableWideStringBuilder : public WideStringBuilderBase -{ -public: - inline ExtendableWideStringBuilder() : WideStringBuilderBase(m_Buffer, N) { m_IsExtendable = true; } - ~ExtendableWideStringBuilder() = default; - -private: - wchar_t m_Buffer[N]; -}; - -template<size_t N> -class WriteToWideString : public ExtendableWideStringBuilder<N> -{ -public: - template<typename... ArgTypes> - explicit WriteToWideString(ArgTypes&&... Args) - { - (*this << ... << Forward<ArgTypes>(Args)); - } -}; - -////////////////////////////////////////////////////////////////////////// - -void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out); -void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out); -void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out); -std::wstring Utf8ToWide(const std::string_view& wstr); - -void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out); -std::string WideToUtf8(const wchar_t* wstr); -void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out); -std::string WideToUtf8(const std::wstring_view Wstr); - -inline uint8_t -Char2Nibble(char c) -{ - if (c >= '0' && c <= '9') - { - return uint8_t(c - '0'); - } - if (c >= 'a' && c <= 'f') - { - return uint8_t(c - 'a' + 10); - } - if (c >= 'A' && c <= 'F') - { - return uint8_t(c - 'A' + 10); - } - return uint8_t(0xff); -}; - -static constexpr const char HexChars[] = "0123456789abcdef"; - -/// <summary> -/// Parse hex string into a byte buffer -/// </summary> -/// <param name="string">Input string</param> -/// <param name="characterCount">Number of characters in string</param> -/// <param name="outPtr">Pointer to output buffer</param> -/// <returns>true if the input consisted of all valid hexadecimal characters</returns> - -inline bool -ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr) -{ - ZEN_ASSERT((CharacterCount & 1) == 0); - - uint8_t allBits = 0; - - while (CharacterCount) - { - uint8_t n0 = Char2Nibble(InputString[0]); - uint8_t n1 = Char2Nibble(InputString[1]); - - allBits |= n0 | n1; - - *OutPtr = (n0 << 4) | n1; - - OutPtr += 1; - InputString += 2; - CharacterCount -= 2; - } - - return (allBits & 0x80) == 0; -} - -inline void -ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString) -{ - while (ByteCount--) - { - uint8_t byte = *InputData++; - - *OutString++ = HexChars[byte >> 4]; - *OutString++ = HexChars[byte & 15]; - } -} - -inline bool -ParseHexNumber(const char* InputString, size_t CharacterCount, uint8_t* OutPtr) -{ - ZEN_ASSERT((CharacterCount & 1) == 0); - - uint8_t allBits = 0; - - InputString += CharacterCount; - while (CharacterCount) - { - InputString -= 2; - uint8_t n0 = Char2Nibble(InputString[0]); - uint8_t n1 = Char2Nibble(InputString[1]); - - allBits |= n0 | n1; - - *OutPtr = (n0 << 4) | n1; - - OutPtr += 1; - CharacterCount -= 2; - } - - return (allBits & 0x80) == 0; -} - -inline void -ToHexNumber(const uint8_t* InputData, size_t ByteCount, char* OutString) -{ - InputData += ByteCount; - while (ByteCount--) - { - uint8_t byte = *(--InputData); - - *OutString++ = HexChars[byte >> 4]; - *OutString++ = HexChars[byte & 15]; - } -} - -/// <summary> -/// Generates a hex number from a pointer to an integer type, this formats the number in the correct order for a hexadecimal number -/// </summary> -/// <param name="Value">Integer value type</param> -/// <param name="outString">Output buffer where resulting string is written</param> -void -ToHexNumber(UnsignedIntegral auto Value, char* OutString) -{ - ToHexNumber((const uint8_t*)&Value, sizeof(Value), OutString); - OutString[sizeof(Value) * 2] = 0; -} - -/// <summary> -/// Parse hex number string into a value, this formats the number in the correct order for a hexadecimal number -/// </summary> -/// <param name="string">Input string</param> -/// <param name="characterCount">Number of characters in string</param> -/// <param name="OutValue">Pointer to output value</param> -/// <returns>true if the input consisted of all valid hexadecimal characters</returns> -bool -ParseHexNumber(const std::string HexString, UnsignedIntegral auto& OutValue) -{ - return ParseHexNumber(HexString.c_str(), sizeof(OutValue) * 2, (uint8_t*)&OutValue); -} - -////////////////////////////////////////////////////////////////////////// -// Format numbers for humans -// - -ZENCORE_API size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer); -ZENCORE_API size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer); -ZENCORE_API size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer); -ZENCORE_API size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer); -ZENCORE_API size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer); - -struct NiceBase -{ - inline const char* c_str() const { return m_Buffer; } - inline operator std::string_view() const { return std::string_view(m_Buffer); } - -protected: - char m_Buffer[16]; -}; - -struct NiceNum : public NiceBase -{ - inline NiceNum(uint64_t Num) { NiceNumToBuffer(Num, m_Buffer); } -}; - -struct NiceBytes : public NiceBase -{ - inline NiceBytes(uint64_t Num) { NiceBytesToBuffer(Num, m_Buffer); } -}; - -struct NiceByteRate : public NiceBase -{ - inline NiceByteRate(uint64_t Bytes, uint64_t TimeMilliseconds) { NiceByteRateToBuffer(Bytes, TimeMilliseconds, m_Buffer); } -}; - -struct NiceLatencyNs : public NiceBase -{ - inline NiceLatencyNs(uint64_t Milliseconds) { NiceLatencyNsToBuffer(Milliseconds, m_Buffer); } -}; - -struct NiceTimeSpanMs : public NiceBase -{ - inline NiceTimeSpanMs(uint64_t Milliseconds) { NiceTimeSpanMsToBuffer(Milliseconds, m_Buffer); } -}; - -////////////////////////////////////////////////////////////////////////// - -inline std::string -NiceRate(uint64_t Num, uint32_t DurationMilliseconds, const char* Unit = "B") -{ - char Buffer[32]; - - if (DurationMilliseconds) - { - // Leave a little of 'Buffer' for the "Unit/s" suffix - std::span<char> BufferSpan(Buffer, sizeof(Buffer) - 8); - NiceNumToBuffer(Num * 1000 / DurationMilliseconds, BufferSpan); - } - else - { - strcpy(Buffer, "0"); - } - - strncat(Buffer, Unit, 4); - strcat(Buffer, "/s"); - - return Buffer; -} - -////////////////////////////////////////////////////////////////////////// - -template<Integral T> -std::optional<T> -ParseInt(const std::string_view& Input) -{ - T Out = 0; - const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out); - if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range) - { - return std::nullopt; - } - return Out; -} - -////////////////////////////////////////////////////////////////////////// - -constexpr uint32_t -HashStringDjb2(const std::string_view& InString) -{ - uint32_t HashValue = 5381; - - for (int CurChar : InString) - { - HashValue = HashValue * 33 + CurChar; - } - - return HashValue; -} - -constexpr uint32_t -HashStringAsLowerDjb2(const std::string_view& InString) -{ - uint32_t HashValue = 5381; - - for (uint8_t CurChar : InString) - { - CurChar -= ((CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic - HashValue = HashValue * 33 + CurChar; - } - - return HashValue; -} - -////////////////////////////////////////////////////////////////////////// - -inline std::string -ToLower(const std::string_view& InString) -{ - std::string Out(InString); - - for (char& CurChar : Out) - { - CurChar -= (uint8_t(CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic - } - - return Out; -} - -////////////////////////////////////////////////////////////////////////// - -template<typename Fn> -uint32_t -ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func) -{ - const char* It = Str.data(); - const char* End = It + Str.length(); - uint32_t Count = 0; - - while (It != End) - { - if (*It == Delim) - { - It++; - continue; - } - - std::string_view Remaining{It, size_t(ptrdiff_t(End - It))}; - size_t Idx = Remaining.find(Delim, 0); - - if (Idx == std::string_view::npos) - { - Idx = Remaining.size(); - } - - Count++; - std::string_view Token{It, Idx}; - if (!Func(Token)) - { - break; - } - - It = It + Idx; - } - - return Count; -} - -////////////////////////////////////////////////////////////////////////// - -inline int32_t -StrCaseCompare(const char* Lhs, const char* Rhs, int64_t Length = -1) -{ - // A helper for cross-platform case-insensitive string comparison. -#if ZEN_PLATFORM_WINDOWS - return (Length < 0) ? _stricmp(Lhs, Rhs) : _strnicmp(Lhs, Rhs, size_t(Length)); -#else - return (Length < 0) ? strcasecmp(Lhs, Rhs) : strncasecmp(Lhs, Rhs, size_t(Length)); -#endif -} - -/** - * @brief - * Helper function to implement case sensitive spaceship operator for strings. - * MacOS clang version we use does not implement <=> for std::string - * @param Lhs string - * @param Rhs string - * @return std::strong_ordering indicating relationship between Lhs and Rhs - */ -inline auto -caseSensitiveCompareStrings(const std::string& Lhs, const std::string& Rhs) -{ - int r = Lhs.compare(Rhs); - return r == 0 ? std::strong_ordering::equal : r < 0 ? std::strong_ordering::less : std::strong_ordering::greater; -} - -////////////////////////////////////////////////////////////////////////// - -/** - * ASCII character bitset useful for fast and readable parsing - * - * Entirely constexpr. Works with both wide and narrow strings. - * - * Example use cases: - * - * constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n"); - * bool bIsWhitespace = WhitespaceCharacters.Test(MyChar); - * const char* HelloWorld = AsciiSet::Skip(" \t\tHello world!", WhitespaceCharacters); - * - * constexpr AsciiSet XmlEscapeChars("&<>\"'"); - * check(AsciiSet::HasNone(EscapedXmlString, XmlEscapeChars)); - * - * constexpr AsciiSet Delimiters(".:;"); - * const TCHAR* DelimiterOrEnd = AsciiSet::FindFirstOrEnd(PrefixedName, Delimiters); - * FString Prefix(PrefixedName, DelimiterOrEnd - PrefixedName); - * - * constexpr AsciiSet Slashes("/\\"); - * const TCHAR* SlashOrEnd = AsciiSet::FindLastOrEnd(PathName, Slashes); - * const TCHAR* FileName = *SlashOrEnd ? SlashOrEnd + 1 : PathName; - */ -class AsciiSet -{ -public: - template<typename CharType, int N> - constexpr AsciiSet(const CharType (&Chars)[N]) : AsciiSet(StringToBitset(Chars)) - { - } - - /** Returns true if a character is part of the set */ - template<typename CharType> - constexpr inline bool Contains(CharType Char) const - { - using UnsignedCharType = typename std::make_unsigned<CharType>::type; - - return !!TestImpl((UnsignedCharType)Char); - } - - /** Returns non-zero if a character is part of the set. Prefer Contains() to avoid VS2019 conversion warnings. */ - template<typename CharType> - constexpr inline uint64_t Test(CharType Char) const - { - using UnsignedCharType = typename std::make_unsigned<CharType>::type; - - return TestImpl((UnsignedCharType)Char); - } - - /** Create new set with specified character in it */ - constexpr inline AsciiSet operator+(char Char) const - { - using UnsignedCharType = typename std::make_unsigned<char>::type; - - InitData Bitset = {LoMask, HiMask}; - SetImpl(Bitset, (UnsignedCharType)Char); - return AsciiSet(Bitset); - } - - /** Create new set containing inverse set of characters - likely including null-terminator */ - constexpr inline AsciiSet operator~() const { return AsciiSet(~LoMask, ~HiMask); } - - ////////// Algorithms for C strings ////////// - - /** Find first character of string inside set or end pointer. Never returns null. */ - template<class CharType> - static constexpr const CharType* FindFirstOrEnd(const CharType* Str, AsciiSet Set) - { - for (AsciiSet SetOrNil(Set.LoMask | NilMask, Set.HiMask); !SetOrNil.Test(*Str); ++Str) - ; - - return Str; - } - - /** Find last character of string inside set or end pointer. Never returns null. */ - template<class CharType> - static constexpr const CharType* FindLastOrEnd(const CharType* Str, AsciiSet Set) - { - const CharType* Last = FindFirstOrEnd(Str, Set); - - for (const CharType* It = Last; *It; It = FindFirstOrEnd(It + 1, Set)) - { - Last = It; - } - - return Last; - } - - /** Find first character of string outside of set. Never returns null. */ - template<typename CharType> - static constexpr const CharType* Skip(const CharType* Str, AsciiSet Set) - { - while (Set.Contains(*Str)) - { - ++Str; - } - - return Str; - } - - /** Test if string contains any character in set */ - template<typename CharType> - static constexpr bool HasAny(const CharType* Str, AsciiSet Set) - { - return *FindFirstOrEnd(Str, Set) != '\0'; - } - - /** Test if string contains no character in set */ - template<typename CharType> - static constexpr bool HasNone(const CharType* Str, AsciiSet Set) - { - return *FindFirstOrEnd(Str, Set) == '\0'; - } - - /** Test if string contains any character outside of set */ - template<typename CharType> - static constexpr bool HasOnly(const CharType* Str, AsciiSet Set) - { - return *Skip(Str, Set) == '\0'; - } - - ////////// Algorithms for string types like std::string_view and std::string ////////// - - /** Get initial substring with all characters in set */ - template<class StringType> - static constexpr StringType FindPrefixWith(const StringType& Str, AsciiSet Set) - { - return Scan<EDir::Forward, EInclude::Members, EKeep::Head>(Str, Set); - } - - /** Get initial substring with no characters in set */ - template<class StringType> - static constexpr StringType FindPrefixWithout(const StringType& Str, AsciiSet Set) - { - return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Head>(Str, Set); - } - - /** Trim initial characters in set */ - template<class StringType> - static constexpr StringType TrimPrefixWith(const StringType& Str, AsciiSet Set) - { - return Scan<EDir::Forward, EInclude::Members, EKeep::Tail>(Str, Set); - } - - /** Trim initial characters not in set */ - template<class StringType> - static constexpr StringType TrimPrefixWithout(const StringType& Str, AsciiSet Set) - { - return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Tail>(Str, Set); - } - - /** Get trailing substring with all characters in set */ - template<class StringType> - static constexpr StringType FindSuffixWith(const StringType& Str, AsciiSet Set) - { - return Scan<EDir::Reverse, EInclude::Members, EKeep::Tail>(Str, Set); - } - - /** Get trailing substring with no characters in set */ - template<class StringType> - static constexpr StringType FindSuffixWithout(const StringType& Str, AsciiSet Set) - { - return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Tail>(Str, Set); - } - - /** Trim trailing characters in set */ - template<class StringType> - static constexpr StringType TrimSuffixWith(const StringType& Str, AsciiSet Set) - { - return Scan<EDir::Reverse, EInclude::Members, EKeep::Head>(Str, Set); - } - - /** Trim trailing characters not in set */ - template<class StringType> - static constexpr StringType TrimSuffixWithout(const StringType& Str, AsciiSet Set) - { - return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Head>(Str, Set); - } - - /** Test if string contains any character in set */ - template<class StringType> - static constexpr bool HasAny(const StringType& Str, AsciiSet Set) - { - return !HasNone(Str, Set); - } - - /** Test if string contains no character in set */ - template<class StringType> - static constexpr bool HasNone(const StringType& Str, AsciiSet Set) - { - uint64_t Match = 0; - for (auto Char : Str) - { - Match |= Set.Test(Char); - } - return Match == 0; - } - - /** Test if string contains any character outside of set */ - template<class StringType> - static constexpr bool HasOnly(const StringType& Str, AsciiSet Set) - { - auto End = Str.data() + Str.size(); - return FindFirst<EInclude::Members>(Set, Str.data(), End) == End; - } - -private: - enum class EDir - { - Forward, - Reverse - }; - enum class EInclude - { - Members, - NonMembers - }; - enum class EKeep - { - Head, - Tail - }; - - template<EInclude Include, typename CharType> - static constexpr const CharType* FindFirst(AsciiSet Set, const CharType* It, const CharType* End) - { - for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); ++It) - ; - return It; - } - - template<EInclude Include, typename CharType> - static constexpr const CharType* FindLast(AsciiSet Set, const CharType* It, const CharType* End) - { - for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); --It) - ; - return It; - } - - template<EDir Dir, EInclude Include, EKeep Keep, class StringType> - static constexpr StringType Scan(const StringType& Str, AsciiSet Set) - { - auto Begin = Str.data(); - auto End = Begin + Str.size(); - auto It = Dir == EDir::Forward ? FindFirst<Include>(Set, Begin, End) : FindLast<Include>(Set, End - 1, Begin - 1) + 1; - - return Keep == EKeep::Head ? StringType(Begin, static_cast<int32_t>(It - Begin)) : StringType(It, static_cast<int32_t>(End - It)); - } - - // Work-around for constexpr limitations - struct InitData - { - uint64_t Lo, Hi; - }; - static constexpr uint64_t NilMask = uint64_t(1) << '\0'; - - static constexpr inline void SetImpl(InitData& Bitset, uint32_t Char) - { - uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0); - uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1); - uint64_t Bit = uint64_t(1) << uint8_t(Char & 0x3f); - - Bitset.Lo |= Bit & IsLo; - Bitset.Hi |= Bit & IsHi; - } - - constexpr inline uint64_t TestImpl(uint32_t Char) const - { - uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0); - uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1); - uint64_t Bit = uint64_t(1) << (Char & 0x3f); - - return (Bit & IsLo & LoMask) | (Bit & IsHi & HiMask); - } - - template<typename CharType, int N> - static constexpr InitData StringToBitset(const CharType (&Chars)[N]) - { - using UnsignedCharType = typename std::make_unsigned<CharType>::type; - - InitData Bitset = {0, 0}; - for (int I = 0; I < N - 1; ++I) - { - SetImpl(Bitset, UnsignedCharType(Chars[I])); - } - - return Bitset; - } - - constexpr AsciiSet(InitData Bitset) : LoMask(Bitset.Lo), HiMask(Bitset.Hi) {} - - constexpr AsciiSet(uint64_t Lo, uint64_t Hi) : LoMask(Lo), HiMask(Hi) {} - - uint64_t LoMask, HiMask; -}; - -////////////////////////////////////////////////////////////////////////// - -void string_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/testing.h b/zencore/include/zencore/testing.h deleted file mode 100644 index a00ee3166..000000000 --- a/zencore/include/zencore/testing.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -#include <memory> - -#ifndef ZEN_TEST_WITH_RUNNER -# define ZEN_TEST_WITH_RUNNER 0 -#endif - -#if ZEN_TEST_WITH_RUNNER -# define DOCTEST_CONFIG_IMPLEMENT -#endif - -#if ZEN_WITH_TESTS -# include <doctest/doctest.h> -inline auto -Approx(auto Value) -{ - return doctest::Approx(Value); -} -#endif - -/** - * Test runner helper - * - * This acts as a thin layer between the test app and the test - * framework, which is used to customize configuration logic - * and to set up logging. - * - * If you don't want to implement custom setup then the - * ZEN_RUN_TESTS macro can be used instead. - */ - -#if ZEN_WITH_TESTS -namespace zen::testing { - -class TestRunner -{ -public: - TestRunner(); - ~TestRunner(); - - int ApplyCommandLine(int argc, char const* const* argv); - int Run(); - -private: - struct Impl; - - std::unique_ptr<Impl> m_Impl; -}; - -# define ZEN_RUN_TESTS(argC, argV) \ - [&] { \ - zen::testing::TestRunner Runner; \ - Runner.ApplyCommandLine(argC, argV); \ - return Runner.Run(); \ - }() - -} // namespace zen::testing -#endif - -#if ZEN_TEST_WITH_RUNNER -# undef DOCTEST_CONFIG_IMPLEMENT -#endif diff --git a/zencore/include/zencore/testutils.h b/zencore/include/zencore/testutils.h deleted file mode 100644 index 04648c6de..000000000 --- a/zencore/include/zencore/testutils.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <filesystem> - -namespace zen { - -std::filesystem::path CreateTemporaryDirectory(); - -class ScopedTemporaryDirectory -{ -public: - explicit ScopedTemporaryDirectory(std::filesystem::path Directory); - ScopedTemporaryDirectory(); - ~ScopedTemporaryDirectory(); - - std::filesystem::path& Path() { return m_RootPath; } - -private: - std::filesystem::path m_RootPath; -}; - -struct ScopedCurrentDirectoryChange -{ - std::filesystem::path OldPath{std::filesystem::current_path()}; - - ScopedCurrentDirectoryChange() { std::filesystem::current_path(CreateTemporaryDirectory()); } - ~ScopedCurrentDirectoryChange() { std::filesystem::current_path(OldPath); } -}; - -} // namespace zen diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h deleted file mode 100644 index a9c96d422..000000000 --- a/zencore/include/zencore/thread.h +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <atomic> -#include <filesystem> -#include <shared_mutex> -#include <string_view> -#include <vector> - -namespace zen { - -void SetCurrentThreadName(std::string_view ThreadName); - -/** - * Reader-writer lock - * - * - A single thread may hold an exclusive lock at any given moment - * - * - Multiple threads may hold shared locks, but only if no thread has - * acquired an exclusive lock - */ -class RwLock -{ -public: - ZENCORE_API void AcquireShared(); - ZENCORE_API void ReleaseShared(); - - ZENCORE_API void AcquireExclusive(); - ZENCORE_API void ReleaseExclusive(); - - struct SharedLockScope - { - SharedLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireShared(); } - ~SharedLockScope() { ReleaseNow(); } - - void ReleaseNow() - { - if (m_Lock) - { - m_Lock->ReleaseShared(); - m_Lock = nullptr; - } - } - - private: - RwLock* m_Lock; - }; - - struct ExclusiveLockScope - { - ExclusiveLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireExclusive(); } - ~ExclusiveLockScope() { ReleaseNow(); } - - void ReleaseNow() - { - if (m_Lock) - { - m_Lock->ReleaseExclusive(); - m_Lock = nullptr; - } - } - - private: - RwLock* m_Lock; - }; - -private: - std::shared_mutex m_Mutex; -}; - -/** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore') - */ -class Event -{ -public: - ZENCORE_API Event(); - ZENCORE_API ~Event(); - - Event(Event&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; } - - Event(const Event& Rhs) = delete; - Event& operator=(const Event& Rhs) = delete; - - inline Event& operator=(Event&& Rhs) noexcept - { - std::swap(m_EventHandle, Rhs.m_EventHandle); - return *this; - } - - ZENCORE_API void Set(); - ZENCORE_API void Reset(); - ZENCORE_API bool Wait(int TimeoutMs = -1); - ZENCORE_API void Close(); - -protected: - explicit Event(void* EventHandle) : m_EventHandle(EventHandle) {} - - void* m_EventHandle = nullptr; -}; - -/** Basic abstraction of an IPC mechanism (aka 'binary semaphore') - */ -class NamedEvent -{ -public: - NamedEvent() = default; - ZENCORE_API explicit NamedEvent(std::string_view EventName); - ZENCORE_API ~NamedEvent(); - ZENCORE_API void Close(); - ZENCORE_API void Set(); - ZENCORE_API bool Wait(int TimeoutMs = -1); - - NamedEvent(NamedEvent&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; } - - inline NamedEvent& operator=(NamedEvent&& Rhs) noexcept - { - std::swap(m_EventHandle, Rhs.m_EventHandle); - return *this; - } - -protected: - void* m_EventHandle = nullptr; - -private: - NamedEvent(const NamedEvent& Rhs) = delete; - NamedEvent& operator=(const NamedEvent& Rhs) = delete; -}; - -/** Basic abstraction of a named (system wide) mutex primitive - */ -class NamedMutex -{ -public: - ~NamedMutex(); - - ZENCORE_API [[nodiscard]] bool Create(std::string_view MutexName); - - ZENCORE_API static bool Exists(std::string_view MutexName); - -private: - void* m_MutexHandle = nullptr; -}; - -/** - * Downward counter of type std::ptrdiff_t which can be used to synchronize threads - */ -class Latch -{ -public: - Latch(std::ptrdiff_t Count) : Counter(Count) {} - - void CountDown() - { - std::ptrdiff_t Old = Counter.fetch_sub(1); - if (Old == 1) - { - Complete.Set(); - } - } - - std::ptrdiff_t Remaining() const { return Counter.load(); } - - // If you want to add dynamic count, make sure to set the initial counter to 1 - // and then do a CountDown() just before wait to not trigger the event causing - // false positive completion results. - void AddCount(std::ptrdiff_t Count) - { - std::atomic_ptrdiff_t Old = Counter.fetch_add(Count); - ZEN_ASSERT_SLOW(Old > 0); - } - - bool Wait(int TimeoutMs = -1) - { - std::ptrdiff_t Old = Counter.load(); - if (Old == 0) - { - return true; - } - return Complete.Wait(TimeoutMs); - } - -private: - std::atomic_ptrdiff_t Counter; - Event Complete; -}; - -/** Basic process abstraction - */ -class ProcessHandle -{ -public: - ZENCORE_API ProcessHandle(); - - ProcessHandle(const ProcessHandle&) = delete; - ProcessHandle& operator=(const ProcessHandle&) = delete; - - ZENCORE_API ~ProcessHandle(); - - ZENCORE_API void Initialize(int Pid); - ZENCORE_API void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle - ZENCORE_API [[nodiscard]] bool IsRunning() const; - ZENCORE_API [[nodiscard]] bool IsValid() const; - ZENCORE_API bool Wait(int TimeoutMs = -1); - ZENCORE_API void Terminate(int ExitCode); - ZENCORE_API void Reset(); - [[nodiscard]] inline int Pid() const { return m_Pid; } - -private: - void* m_ProcessHandle = nullptr; - int m_Pid = 0; -}; - -/** Basic process creation - */ -struct CreateProcOptions -{ - enum - { - Flag_NewConsole = 1 << 0, - Flag_Elevated = 1 << 1, - Flag_Unelevated = 1 << 2, - }; - - const std::filesystem::path* WorkingDirectory = nullptr; - uint32_t Flags = 0; -}; - -#if ZEN_PLATFORM_WINDOWS -using CreateProcResult = void*; // handle to the process -#else -using CreateProcResult = int32_t; // pid -#endif - -ZENCORE_API CreateProcResult CreateProc(const std::filesystem::path& Executable, - std::string_view CommandLine, // should also include arg[0] (executable name) - const CreateProcOptions& Options = {}); - -/** Process monitor - monitors a list of running processes via polling - - Intended to be used to monitor a set of "sponsor" processes, where - we need to determine when none of them remain alive - - */ - -class ProcessMonitor -{ -public: - ProcessMonitor(); - ~ProcessMonitor(); - - ZENCORE_API bool IsRunning(); - ZENCORE_API void AddPid(int Pid); - ZENCORE_API bool IsActive() const; - -private: - using HandleType = void*; - - mutable RwLock m_Lock; - std::vector<HandleType> m_ProcessHandles; -}; - -ZENCORE_API bool IsProcessRunning(int pid); -ZENCORE_API int GetCurrentProcessId(); -ZENCORE_API int GetCurrentThreadId(); - -ZENCORE_API void Sleep(int ms); - -void thread_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/timer.h b/zencore/include/zencore/timer.h deleted file mode 100644 index e4ddc3505..000000000 --- a/zencore/include/zencore/timer.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#if ZEN_COMPILER_MSC -# include <intrin.h> -#elif ZEN_ARCH_X64 -# include <x86intrin.h> -#endif - -#include <stdint.h> - -namespace zen { - -// High frequency timers - -ZENCORE_API uint64_t GetHifreqTimerValue(); -ZENCORE_API uint64_t GetHifreqTimerFrequency(); -ZENCORE_API double GetHifreqTimerToSeconds(); -ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init - -class Stopwatch -{ -public: - inline Stopwatch() : m_StartValue(GetHifreqTimerValue()) {} - - inline uint64_t GetElapsedTimeMs() const { return (GetHifreqTimerValue() - m_StartValue) * 1'000 / GetHifreqTimerFrequency(); } - inline uint64_t GetElapsedTimeUs() const { return (GetHifreqTimerValue() - m_StartValue) * 1'000'000 / GetHifreqTimerFrequency(); } - inline uint64_t GetElapsedTicks() const { return GetHifreqTimerValue() - m_StartValue; } - inline void Reset() { m_StartValue = GetHifreqTimerValue(); } - - static inline uint64_t GetElapsedTimeMs(uint64_t Ticks) { return Ticks * 1'000 / GetHifreqTimerFrequency(); } - static inline uint64_t GetElapsedTimeUs(uint64_t Ticks) { return Ticks * 1'000'000 / GetHifreqTimerFrequency(); } - -private: - uint64_t m_StartValue; -}; - -// Low frequency timers - -namespace detail { - extern ZENCORE_API uint64_t g_LofreqTimerValue; -} // namespace detail - -inline uint64_t -GetLofreqTimerValue() -{ - return detail::g_LofreqTimerValue; -} - -ZENCORE_API void UpdateLofreqTimerValue(); -ZENCORE_API uint64_t GetLofreqTimerFrequency(); - -void timer_forcelink(); // internal - -} // namespace zen diff --git a/zencore/include/zencore/trace.h b/zencore/include/zencore/trace.h deleted file mode 100644 index 0af490f23..000000000 --- a/zencore/include/zencore/trace.h +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -/* clang-format off */ - -#include <zencore/zencore.h> - -#if ZEN_WITH_TRACE - -ZEN_THIRD_PARTY_INCLUDES_START -#if !defined(TRACE_IMPLEMENT) -# define TRACE_IMPLEMENT 0 -#endif -#include <trace.h> -#undef TRACE_IMPLEMENT -ZEN_THIRD_PARTY_INCLUDES_END - -#define ZEN_TRACE_CPU(x) TRACE_CPU_SCOPE(x) - -enum class TraceType -{ - File, - Network, - None -}; - -void TraceInit(const char* HostOrPath, TraceType Type); - -#else - -#define ZEN_TRACE_CPU(x) - -#endif // ZEN_WITH_TRACE - -/* clang-format on */ diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h deleted file mode 100644 index 9659f5893..000000000 --- a/zencore/include/zencore/uid.h +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> -#include <compare> - -namespace zen { - -class StringBuilderBase; - -/** Object identifier - - Can be used as a GUID essentially, but is more compact (12 bytes) and as such - is more susceptible to collisions than a 16-byte GUID but also I don't expect - the population to be large so in practice the risk should be minimal due to - how the identifiers work. - - Similar in spirit to MongoDB ObjectId - - When serialized, object identifiers generated in a given session in sequence - will sort in chronological order since the timestamp is in the MSB in big - endian format. This makes it suitable as a database key since most indexing - structures work better when keys are inserted in lexicographically - increasing order. - - The current layout is basically: - - |----------------|----------------|----------------| - | timestamp | serial # | run id | - |----------------|----------------|----------------| - MSB LSB - - - Timestamp is a unsigned 32-bit value (seconds since 00:00:00 Jan 1 2021) - - Serial # is another unsigned 32-bit value which is assigned a (strong) - random number at initialization time which is incremented when a new Oid - is generated - - The run id is generated from a strong random number generator - at initialization time and stays fixed for the duration of the program - - Timestamp and serial are stored in memory in such a way that they can be - ordered lexicographically. I.e they are in big-endian byte order. - - NOTE: The information above is only meant to explain the properties of - the identifiers. Client code should simply treat the identifier as an - opaque value and may not make any assumptions on the structure, as there - may be other ways of generating the identifiers in the future if an - application benefits. - - */ - -struct Oid -{ - static const int StringLength = 24; - typedef char String_t[StringLength + 1]; - - static void Initialize(); - [[nodiscard]] static Oid NewOid(); - - const Oid& Generate(); - [[nodiscard]] static Oid FromHexString(const std::string_view String); - StringBuilderBase& ToString(StringBuilderBase& OutString) const; - void ToString(char OutString[StringLength]); - [[nodiscard]] static Oid FromMemory(const void* Ptr); - - auto operator<=>(const Oid& rhs) const = default; - [[nodiscard]] inline operator bool() const { return *this != Zero; } - - static const Oid Zero; // Min (can be used to signify a "null" value, or for open range queries) - static const Oid Max; // Max (can be used for open range queries) - - struct Hasher - { - size_t operator()(const Oid& id) const - { - const size_t seed = id.OidBits[0]; - return ((seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1])) | (uint64_t(id.OidBits[2]) << 32); - } - }; - - // You should not assume anything about these words - uint32_t OidBits[3]; -}; - -extern void uid_forcelink(); - -} // namespace zen diff --git a/zencore/include/zencore/varint.h b/zencore/include/zencore/varint.h deleted file mode 100644 index e57e1d497..000000000 --- a/zencore/include/zencore/varint.h +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "intmath.h" - -#include <algorithm> - -namespace zen { - -// Variable-Length Integer Encoding -// -// ZigZag encoding is used to convert signed integers into unsigned integers in a way that allows -// integers with a small magnitude to have a smaller encoded representation. -// -// An unsigned integer is encoded into 1-9 bytes based on its magnitude. The first byte indicates -// how many additional bytes are used by the number of leading 1-bits that it has. The additional -// bytes are stored in big endian order, and the most significant bits of the value are stored in -// the remaining bits in the first byte. The encoding of the first byte allows the reader to skip -// over the encoded integer without consuming its bytes individually. -// -// Encoded unsigned integers sort the same in a byte-wise comparison as when their decoded values -// are compared. The same property does not hold for signed integers due to ZigZag encoding. -// -// 32-bit inputs encode to 1-5 bytes. -// 64-bit inputs encode to 1-9 bytes. -// -// 0x0000'0000'0000'0000 - 0x0000'0000'0000'007f : 0b0_______ 1 byte -// 0x0000'0000'0000'0080 - 0x0000'0000'0000'3fff : 0b10______ 2 bytes -// 0x0000'0000'0000'4000 - 0x0000'0000'001f'ffff : 0b110_____ 3 bytes -// 0x0000'0000'0020'0000 - 0x0000'0000'0fff'ffff : 0b1110____ 4 bytes -// 0x0000'0000'1000'0000 - 0x0000'0007'ffff'ffff : 0b11110___ 5 bytes -// 0x0000'0008'0000'0000 - 0x0000'03ff'ffff'ffff : 0b111110__ 6 bytes -// 0x0000'0400'0000'0000 - 0x0001'ffff'ffff'ffff : 0b1111110_ 7 bytes -// 0x0002'0000'0000'0000 - 0x00ff'ffff'ffff'ffff : 0b11111110 8 bytes -// 0x0100'0000'0000'0000 - 0xffff'ffff'ffff'ffff : 0b11111111 9 bytes -// -// Encoding Examples -// -42 => ZigZag => 0x53 => 0x53 -// 42 => ZigZag => 0x54 => 0x54 -// 0x1 => 0x01 -// 0x12 => 0x12 -// 0x123 => 0x81 0x23 -// 0x1234 => 0x92 0x34 -// 0x12345 => 0xc1 0x23 0x45 -// 0x123456 => 0xd2 0x34 0x56 -// 0x1234567 => 0xe1 0x23 0x45 0x67 -// 0x12345678 => 0xf0 0x12 0x34 0x56 0x78 -// 0x123456789 => 0xf1 0x23 0x45 0x67 0x89 -// 0x123456789a => 0xf8 0x12 0x34 0x56 0x78 0x9a -// 0x123456789ab => 0xfb 0x23 0x45 0x67 0x89 0xab -// 0x123456789abc => 0xfc 0x12 0x34 0x56 0x78 0x9a 0xbc -// 0x123456789abcd => 0xfd 0x23 0x45 0x67 0x89 0xab 0xcd -// 0x123456789abcde => 0xfe 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde -// 0x123456789abcdef => 0xff 0x01 0x23 0x45 0x67 0x89 0xab 0xcd 0xef -// 0x123456789abcdef0 => 0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0 - -/** - * Measure the length in bytes (1-9) of an encoded variable-length integer. - * - * @param InData A variable-length encoding of an (signed or unsigned) integer. - * @return The number of bytes used to encode the integer, in the range 1-9. - */ -inline uint32_t -MeasureVarUInt(const void* InData) -{ - return CountLeadingZeros(uint8_t(~*static_cast<const uint8_t*>(InData))) - 23; -} - -/** Measure the length in bytes (1-9) of an encoded variable-length integer. \see \ref MeasureVarUInt */ -inline uint32_t -MeasureVarInt(const void* InData) -{ - return MeasureVarUInt(InData); -} - -/** Measure the number of bytes (1-5) required to encode the 32-bit input. */ -inline uint32_t -MeasureVarUInt(uint32_t InValue) -{ - return uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1); -} - -/** Measure the number of bytes (1-9) required to encode the 64-bit input. */ -inline uint32_t -MeasureVarUInt(uint64_t InValue) -{ - return uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9)); -} - -/** Measure the number of bytes (1-5) required to encode the 32-bit input. \see \ref MeasureVarUInt */ -inline uint32_t -MeasureVarInt(int32_t InValue) -{ - return MeasureVarUInt(uint32_t((InValue >> 31) ^ (InValue << 1))); -} - -/** Measure the number of bytes (1-9) required to encode the 64-bit input. \see \ref MeasureVarUInt */ -inline uint32_t -MeasureVarInt(int64_t InValue) -{ - return MeasureVarUInt(uint64_t((InValue >> 63) ^ (InValue << 1))); -} - -/** - * Read a variable-length unsigned integer. - * - * @param InData A variable-length encoding of an unsigned integer. - * @param OutByteCount The number of bytes consumed from the input. - * @return An unsigned integer. - */ -inline uint64_t -ReadVarUInt(const void* InData, uint32_t& OutByteCount) -{ - const uint32_t ByteCount = MeasureVarUInt(InData); - OutByteCount = ByteCount; - - const uint8_t* InBytes = static_cast<const uint8_t*>(InData); - uint64_t Value = *InBytes++ & uint8_t(0xff >> ByteCount); - switch (ByteCount - 1) - { - case 8: - Value <<= 8; - Value |= *InBytes++; - [[fallthrough]]; - case 7: - Value <<= 8; - Value |= *InBytes++; - [[fallthrough]]; - case 6: - Value <<= 8; - Value |= *InBytes++; - [[fallthrough]]; - case 5: - Value <<= 8; - Value |= *InBytes++; - [[fallthrough]]; - case 4: - Value <<= 8; - Value |= *InBytes++; - [[fallthrough]]; - case 3: - Value <<= 8; - Value |= *InBytes++; - [[fallthrough]]; - case 2: - Value <<= 8; - Value |= *InBytes++; - [[fallthrough]]; - case 1: - Value <<= 8; - Value |= *InBytes++; - [[fallthrough]]; - default: - return Value; - } -} - -/** - * Read a variable-length signed integer. - * - * @param InData A variable-length encoding of a signed integer. - * @param OutByteCount The number of bytes consumed from the input. - * @return A signed integer. - */ -inline int64_t -ReadVarInt(const void* InData, uint32_t& OutByteCount) -{ - const uint64_t Value = ReadVarUInt(InData, OutByteCount); - return -int64_t(Value & 1) ^ int64_t(Value >> 1); -} - -/** - * Write a variable-length unsigned integer. - * - * @param InValue An unsigned integer to encode. - * @param OutData A buffer of at least 5 bytes to write the output to. - * @return The number of bytes used in the output. - */ -inline uint32_t -WriteVarUInt(uint32_t InValue, void* OutData) -{ - const uint32_t ByteCount = MeasureVarUInt(InValue); - uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1; - switch (ByteCount - 1) - { - case 4: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 3: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 2: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 1: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - default: - break; - } - *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue); - return ByteCount; -} - -/** - * Write a variable-length unsigned integer. - * - * @param InValue An unsigned integer to encode. - * @param OutData A buffer of at least 9 bytes to write the output to. - * @return The number of bytes used in the output. - */ -inline uint32_t -WriteVarUInt(uint64_t InValue, void* OutData) -{ - const uint32_t ByteCount = MeasureVarUInt(InValue); - uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1; - switch (ByteCount - 1) - { - case 8: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 7: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 6: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 5: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 4: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 3: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 2: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - case 1: - *OutBytes-- = uint8_t(InValue); - InValue >>= 8; - [[fallthrough]]; - default: - break; - } - *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue); - return ByteCount; -} - -/** Write a variable-length signed integer. \see \ref WriteVarUInt */ -inline uint32_t -WriteVarInt(int32_t InValue, void* OutData) -{ - const uint32_t Value = uint32_t((InValue >> 31) ^ (InValue << 1)); - return WriteVarUInt(Value, OutData); -} - -/** Write a variable-length signed integer. \see \ref WriteVarUInt */ -inline uint32_t -WriteVarInt(int64_t InValue, void* OutData) -{ - const uint64_t Value = uint64_t((InValue >> 63) ^ (InValue << 1)); - return WriteVarUInt(Value, OutData); -} - -} // namespace zen diff --git a/zencore/include/zencore/windows.h b/zencore/include/zencore/windows.h deleted file mode 100644 index 91828f0ec..000000000 --- a/zencore/include/zencore/windows.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -ZEN_THIRD_PARTY_INCLUDES_START - -struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- -#ifndef NOMINMAX -# define NOMINMAX // We don't want your min/max macros -#endif -#ifndef NOGDI -# define NOGDI // We don't want your GetObject define -#endif -#ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -#endif -#ifndef _WIN32_WINNT -# define _WIN32_WINNT 0x0A00 -#endif -#include <windows.h> -#undef GetObject - -ZEN_THIRD_PARTY_INCLUDES_END diff --git a/zencore/include/zencore/workthreadpool.h b/zencore/include/zencore/workthreadpool.h deleted file mode 100644 index 0ddc65298..000000000 --- a/zencore/include/zencore/workthreadpool.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -#include <zencore/blockingqueue.h> -#include <zencore/refcount.h> - -#include <exception> -#include <functional> -#include <system_error> -#include <thread> -#include <vector> - -namespace zen { - -struct IWork : public RefCounted -{ - virtual void Execute() = 0; - - inline std::exception_ptr GetException() { return m_Exception; } - -private: - std::exception_ptr m_Exception; - - friend class WorkerThreadPool; -}; - -class WorkerThreadPool -{ -public: - WorkerThreadPool(int InThreadCount); - ~WorkerThreadPool(); - - void ScheduleWork(Ref<IWork> Work); - void ScheduleWork(std::function<void()>&& Work); - - [[nodiscard]] size_t PendingWork() const; - -private: - void WorkerThreadFunction(); - - std::vector<std::thread> m_WorkerThreads; - BlockingQueue<Ref<IWork>> m_WorkQueue; -}; - -} // namespace zen diff --git a/zencore/include/zencore/xxhash.h b/zencore/include/zencore/xxhash.h deleted file mode 100644 index 04872f4c3..000000000 --- a/zencore/include/zencore/xxhash.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <zencore/memory.h> - -#include <xxh3.h> - -#include <compare> -#include <string_view> - -namespace zen { - -class StringBuilderBase; - -/** - * XXH3 hash - */ -struct XXH3_128 -{ - uint8_t Hash[16]; - - static XXH3_128 MakeFrom(const void* data /* 16 bytes */) - { - XXH3_128 Xx; - memcpy(Xx.Hash, data, sizeof Xx); - return Xx; - } - - static inline XXH3_128 HashMemory(const void* data, size_t byteCount) - { - XXH3_128 Hash; - XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits(data, byteCount)); - return Hash; - } - static XXH3_128 HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); } - static XXH3_128 FromHexString(const char* string); - static XXH3_128 FromHexString(const std::string_view string); - const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const; - StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const; - - static const int StringLength = 32; - typedef char String_t[StringLength + 1]; - - static XXH3_128 Zero; // Initialized to all zeros - - inline auto operator<=>(const XXH3_128& rhs) const = default; - - struct Hasher - { - size_t operator()(const XXH3_128& v) const - { - size_t h; - memcpy(&h, v.Hash, sizeof h); - return h; - } - }; -}; - -struct XXH3_128Stream -{ - /// Begin streaming hash compute (not needed on freshly constructed instance) - void Reset() { memset(&m_State, 0, sizeof m_State); } - - /// Append another chunk - XXH3_128Stream& Append(const void* Data, size_t ByteCount) - { - XXH3_128bits_update(&m_State, Data, ByteCount); - return *this; - } - - /// Append another chunk - XXH3_128Stream& Append(MemoryView Data) { return Append(Data.GetData(), Data.GetSize()); } - - /// Obtain final hash. If you wish to reuse the instance call reset() - XXH3_128 GetHash() - { - XXH3_128 Hash; - XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits_digest(&m_State)); - return Hash; - } - -private: - XXH3_state_s m_State{}; -}; - -} // namespace zen diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h deleted file mode 100644 index 5bcd77239..000000000 --- a/zencore/include/zencore/zencore.h +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <cinttypes> -#include <stdexcept> -#include <string> -#include <version> - -#ifndef ZEN_WITH_TESTS -# define ZEN_WITH_TESTS 1 -#endif - -////////////////////////////////////////////////////////////////////////// -// Platform -// - -#define ZEN_PLATFORM_WINDOWS 0 -#define ZEN_PLATFORM_LINUX 0 -#define ZEN_PLATFORM_MAC 0 - -#ifdef _WIN32 -# undef ZEN_PLATFORM_WINDOWS -# define ZEN_PLATFORM_WINDOWS 1 -#elif defined(__linux__) -# undef ZEN_PLATFORM_LINUX -# define ZEN_PLATFORM_LINUX 1 -#elif defined(__APPLE__) -# undef ZEN_PLATFORM_MAC -# define ZEN_PLATFORM_MAC 1 -#endif - -#if ZEN_PLATFORM_WINDOWS -# if !defined(NOMINMAX) -# define NOMINMAX // stops Windows.h from defining 'min/max' macros -# endif -# if !defined(NOGDI) -# define NOGDI -# endif -# if !defined(WIN32_LEAN_AND_MEAN) -# define WIN32_LEAN_AND_MEAN // cut-down what Windows.h defines -# endif -#endif - -////////////////////////////////////////////////////////////////////////// -// Compiler -// - -#define ZEN_COMPILER_CLANG 0 -#define ZEN_COMPILER_MSC 0 -#define ZEN_COMPILER_GCC 0 - -// Clang can define __GNUC__ and/or _MSC_VER so we check for Clang first -#ifdef __clang__ -# undef ZEN_COMPILER_CLANG -# define ZEN_COMPILER_CLANG 1 -#elif defined(_MSC_VER) -# undef ZEN_COMPILER_MSC -# define ZEN_COMPILER_MSC 1 -#elif defined(__GNUC__) -# undef ZEN_COMPILER_GCC -# define ZEN_COMPILER_GCC 1 -#else -# error Unknown compiler -#endif - -#if ZEN_COMPILER_MSC -# pragma warning(disable : 4324) // warning C4324: '<type>': structure was padded due to alignment specifier -# pragma warning(default : 4668) // warning C4668: 'symbol' is not defined as a preprocessor macro, replacing with '0' for 'directives' -# pragma warning(default : 4100) // warning C4100: 'identifier' : unreferenced formal parameter -#endif - -#ifndef ZEN_THIRD_PARTY_INCLUDES_START -# if ZEN_COMPILER_MSC -# define ZEN_THIRD_PARTY_INCLUDES_START \ - __pragma(warning(push)) __pragma(warning(disable : 4668)) /* C4668: use of undefined preprocessor macro */ \ - __pragma(warning(disable : 4305)) /* C4305: 'if': truncation from 'uint32' to 'bool' */ \ - __pragma(warning(disable : 4267)) /* C4267: '=': conversion from 'size_t' to 'US' */ \ - __pragma(warning(disable : 4127)) /* C4127: conditional expression is constant */ \ - __pragma(warning(disable : 4189)) /* C4189: local variable is initialized but not referenced */ -# elif ZEN_COMPILER_CLANG -# define ZEN_THIRD_PARTY_INCLUDES_START \ - _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wundef\"") \ - _Pragma("clang diagnostic ignored \"-Wunused-parameter\"") _Pragma("clang diagnostic ignored \"-Wunused-variable\"") -# elif ZEN_COMPILER_GCC -# define ZEN_THIRD_PARTY_INCLUDES_START \ - _Pragma("GCC diagnostic push") /* NB. ignoring -Wundef doesn't work with GCC */ \ - _Pragma("GCC diagnostic ignored \"-Wunused-parameter\"") _Pragma("GCC diagnostic ignored \"-Wunused-variable\"") -# endif -#endif - -#ifndef ZEN_THIRD_PARTY_INCLUDES_END -# if ZEN_COMPILER_MSC -# define ZEN_THIRD_PARTY_INCLUDES_END __pragma(warning(pop)) -# elif ZEN_COMPILER_CLANG -# define ZEN_THIRD_PARTY_INCLUDES_END _Pragma("clang diagnostic pop") -# elif ZEN_COMPILER_GCC -# define ZEN_THIRD_PARTY_INCLUDES_END _Pragma("GCC diagnostic pop") -# endif -#endif - -#if ZEN_COMPILER_MSC -# define ZEN_DEBUG_BREAK() \ - do \ - { \ - __debugbreak(); \ - } while (0) -#else -# define ZEN_DEBUG_BREAK() \ - do \ - { \ - __builtin_trap(); \ - } while (0) -#endif - -////////////////////////////////////////////////////////////////////////// -// C++20 support -// - -// Clang -#if ZEN_COMPILER_CLANG && __clang_major__ < 12 -# error clang-12 onwards is required for C++20 support -#endif - -// GCC -#if ZEN_COMPILER_GCC && __GNUC__ < 11 -# error GCC-11 onwards is required for C++20 support -#endif - -// GNU libstdc++ -#if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11 -# error GNU libstdc++-11 onwards is required for C++20 support -#endif - -// LLVM libc++ -#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 12000 -# error LLVM libc++-12 onwards is required for C++20 support -#endif - -// At the time of writing only ver >= 13 of LLVM's libc++ has an implementation -// of std::integral. Some platforms like Ubuntu and Mac OS are still on 12. -#if defined(__cpp_lib_concepts) -# include <concepts> -template<class T> -concept Integral = std::integral<T>; -template<class T> -concept SignedIntegral = std::signed_integral<T>; -template<class T> -concept UnsignedIntegral = std::unsigned_integral<T>; -template<class F, class... A> -concept Invocable = std::invocable<F, A...>; -template<class D, class B> -concept DerivedFrom = std::derived_from<D, B>; -#else -template<class T> -concept Integral = std::is_integral_v<T>; -template<class T> -concept SignedIntegral = Integral<T> && std::is_signed_v<T>; -template<class T> -concept UnsignedIntegral = Integral<T> && !std::is_signed_v<T>; -template<class F, class... A> -concept Invocable = requires(F&& f, A&&... a) -{ - std::invoke(std::forward<F>(f), std::forward<A>(a)...); -}; -template<class D, class B> -concept DerivedFrom = std::is_base_of_v<B, D> && std::is_convertible_v<const volatile D*, const volatile B*>; -#endif - -#if defined(__cpp_lib_ranges) -template<typename T> -concept ContiguousRange = std::ranges::contiguous_range<T>; -#else -template<typename T> -concept ContiguousRange = true; -#endif - -////////////////////////////////////////////////////////////////////////// -// Architecture -// - -#if defined(__amd64__) || defined(_M_X64) -# define ZEN_ARCH_X64 1 -# define ZEN_ARCH_ARM64 0 -#elif defined(__arm64__) || defined(_M_ARM64) -# define ZEN_ARCH_X64 0 -# define ZEN_ARCH_ARM64 1 -#else -# error Unknown architecture -#endif - -////////////////////////////////////////////////////////////////////////// -// Build flavor -// - -#ifdef NDEBUG -# define ZEN_BUILD_DEBUG 0 -# define ZEN_BUILD_RELEASE 1 -#else -# define ZEN_BUILD_DEBUG 1 -# define ZEN_BUILD_RELEASE 0 -#endif - -////////////////////////////////////////////////////////////////////////// - -#define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1 - -#if defined(__SIZEOF_WCHAR_T__) && __SIZEOF_WCHAR_T__ == 4 -# define ZEN_SIZEOF_WCHAR_T 4 -#else -static_assert(sizeof(wchar_t) == 2, "wchar_t is expected to be two bytes in size"); -# define ZEN_SIZEOF_WCHAR_T 2 -#endif - -////////////////////////////////////////////////////////////////////////// -// Assert -// - -#if ZEN_PLATFORM_WINDOWS -// Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable. -# define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name)) -# define ZEN_FORCENOINLINE __declspec(noinline) /* Force code to NOT be inline */ -# define LINE_TERMINATOR_ANSI "\r\n" -#else -# define ZEN_CODE_SECTION(Name) -# define ZEN_FORCENOINLINE -# define LINE_TERMINATOR_ANSI "\n" -#endif - -#if ZEN_ARCH_ARM64 -// On ARM we can't do this because the executable will require jumps larger -// than the branch instruction can handle. Clang will only generate -// the trampolines in the .text segment of the binary. If the zcold segment -// is present it will generate code that it cannot link. -# define ZEN_DEBUG_SECTION -#else -// We'll put all assert implementation code into a separate section in the linked -// executable. This code should never execute so using a separate section keeps -// it well off the hot path and hopefully out of the instruction cache. It also -// facilitates reasoning about the makeup of a compiled/linked binary. -# define ZEN_DEBUG_SECTION ZEN_CODE_SECTION(".zcold") -#endif - -namespace zen -{ - class AssertException : public std::logic_error - { - public: - AssertException(const char* Msg) : std::logic_error(Msg) {} - }; - - struct AssertImpl - { - static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecAssert - [[noreturn]] (const char* Filename, int LineNumber, const char* FunctionName, const char* Msg) - { - CurrentAssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg); - throw AssertException{Msg}; - } - - protected: - virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION OnAssert(const char* Filename, - int LineNumber, - const char* FunctionName, - const char* Msg) - { - (void(Filename)); - (void(LineNumber)); - (void(FunctionName)); - (void(Msg)); - } - static AssertImpl DefaultAssertImpl; - static AssertImpl* CurrentAssertImpl; - }; - -} // namespace zen - -#define ZEN_ASSERT(x, ...) \ - do \ - { \ - if (x) [[unlikely]] \ - break; \ - zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ - } while (false) - -#ifndef NDEBUG -# define ZEN_ASSERT_SLOW(x, ...) \ - do \ - { \ - if (x) [[unlikely]] \ - break; \ - zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ - } while (false) -#else -# define ZEN_ASSERT_SLOW(x, ...) -#endif - -////////////////////////////////////////////////////////////////////////// - -#ifdef __clang__ -template<typename T> -auto ZenArrayCountHelper(T& t) -> typename std::enable_if<__is_array(T), char (&)[sizeof(t) / sizeof(t[0]) + 1]>::type; -#else -template<typename T, uint32_t N> -char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; -#endif - -#define ZEN_ARRAY_COUNT(array) (sizeof(ZenArrayCountHelper(array)) - 1) - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_COMPILER_MSC -# define ZEN_NOINLINE __declspec(noinline) -#else -# define ZEN_NOINLINE __attribute__((noinline)) -#endif - -#if ZEN_PLATFORM_WINDOWS -# define ZEN_EXE_SUFFIX_LITERAL ".exe" -#else -# define ZEN_EXE_SUFFIX_LITERAL -#endif - -#define ZEN_UNUSED(...) ((void)__VA_ARGS__) -#define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__) -#define ZENCORE_API // Placeholder to allow DLL configs in the future (maybe) - -namespace zen { - -ZENCORE_API bool IsApplicationExitRequested(); -ZENCORE_API void RequestApplicationExit(int ExitCode); -ZENCORE_API bool IsDebuggerPresent(); -ZENCORE_API void SetIsInteractiveSession(bool Value); -ZENCORE_API bool IsInteractiveSession(); - -ZENCORE_API void zencore_forcelinktests(); - -} // namespace zen - -////////////////////////////////////////////////////////////////////////// - -#ifndef ZEN_USE_MIMALLOC -# if ZEN_ARCH_ARM64 - // The vcpkg mimalloc port doesn't support Arm targets -# define ZEN_USE_MIMALLOC 0 -# else -# define ZEN_USE_MIMALLOC 1 -# endif -#endif - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_COMPILER_MSC -# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) -# define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on)) -#elif ZEN_COMPILER_GCC -# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("GCC push_options") _Pragma("GCC optimize (\"O0\")") -# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("GCC pop_options") -#elif ZEN_COMPILER_CLANG -# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize off") -# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize on") -#endif - -// Set up optimization control macros, now that we have both the build settings and the platform macros -#define ZEN_DISABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL - -#if ZEN_BUILD_DEBUG -# define ZEN_ENABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL -#else -# define ZEN_ENABLE_OPTIMIZATION ZEN_ENABLE_OPTIMIZATION_ACTUAL -#endif - -#define ZEN_ENABLE_OPTIMIZATION_ALWAYS ZEN_ENABLE_OPTIMIZATION_ACTUAL - -////////////////////////////////////////////////////////////////////////// - -#ifndef ZEN_WITH_TRACE -# define ZEN_WITH_TRACE 0 -#endif - -////////////////////////////////////////////////////////////////////////// - -using ThreadId_t = uint32_t; diff --git a/zencore/intmath.cpp b/zencore/intmath.cpp deleted file mode 100644 index 5a686dc8e..000000000 --- a/zencore/intmath.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/endian.h> -#include <zencore/intmath.h> - -#include <zencore/testing.h> - -namespace zen { - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -void -intmath_forcelink() -{ -} - -TEST_CASE("intmath") -{ - CHECK(FloorLog2(0x00) == 0); - CHECK(FloorLog2(0x01) == 0); - CHECK(FloorLog2(0x0f) == 3); - CHECK(FloorLog2(0x10) == 4); - CHECK(FloorLog2(0x11) == 4); - CHECK(FloorLog2(0x12) == 4); - CHECK(FloorLog2(0x22) == 5); - CHECK(FloorLog2(0x0001'0000) == 16); - CHECK(FloorLog2(0x0001'000f) == 16); - CHECK(FloorLog2(0x8000'0000) == 31); - - CHECK(FloorLog2_64(0x00ull) == 0); - CHECK(FloorLog2_64(0x01ull) == 0); - CHECK(FloorLog2_64(0x0full) == 3); - CHECK(FloorLog2_64(0x10ull) == 4); - CHECK(FloorLog2_64(0x11ull) == 4); - CHECK(FloorLog2_64(0x0001'0000ull) == 16); - CHECK(FloorLog2_64(0x0001'000full) == 16); - CHECK(FloorLog2_64(0x8000'0000ull) == 31); - CHECK(FloorLog2_64(0x0000'0001'0000'0000ull) == 32); - CHECK(FloorLog2_64(0x8000'0000'0000'0000ull) == 63); - - CHECK(CountLeadingZeros64(0x8000'0000'0000'0000ull) == 0); - CHECK(CountLeadingZeros64(0x0000'0000'0000'0000ull) == 64); - CHECK(CountLeadingZeros64(0x0000'0000'0000'0001ull) == 63); - CHECK(CountLeadingZeros64(0x0000'0000'8000'0000ull) == 32); - CHECK(CountLeadingZeros64(0x0000'0001'0000'0000ull) == 31); - - CHECK(CountTrailingZeros64(0x8000'0000'0000'0000ull) == 63); - CHECK(CountTrailingZeros64(0x0000'0000'0000'0000ull) == 64); - CHECK(CountTrailingZeros64(0x0000'0000'0000'0001ull) == 0); - CHECK(CountTrailingZeros64(0x0000'0000'8000'0000ull) == 31); - CHECK(CountTrailingZeros64(0x0000'0001'0000'0000ull) == 32); - - CHECK(ByteSwap(uint16_t(0x6d72)) == 0x726d); - CHECK(ByteSwap(uint32_t(0x2741'3965)) == 0x6539'4127); - CHECK(ByteSwap(uint64_t(0x214d'6172'7469'6e21ull)) == 0x216e'6974'7261'4d21ull); -} - -#endif - -} // namespace zen diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp deleted file mode 100644 index 1d7d47695..000000000 --- a/zencore/iobuffer.cpp +++ /dev/null @@ -1,653 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/iobuffer.h> - -#include <zencore/except.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/iohash.h> -#include <zencore/logging.h> -#include <zencore/memory.h> -#include <zencore/testing.h> -#include <zencore/thread.h> - -#include <memory.h> -#include <system_error> - -#if ZEN_USE_MIMALLOC -ZEN_THIRD_PARTY_INCLUDES_START -# include <mimalloc.h> -ZEN_THIRD_PARTY_INCLUDES_END -#endif - -#if ZEN_PLATFORM_WINDOWS -# include <atlfile.h> -#else -# include <sys/stat.h> -# include <sys/mman.h> -#endif - -#include <gsl/gsl-lite.hpp> - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -void -IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment) const -{ -#if ZEN_PLATFORM_WINDOWS - if (((InSize & 0xffFF) == 0) && (Alignment == 0x10000)) - { - m_Flags.fetch_or(kLowLevelAlloc, std::memory_order_relaxed); - m_DataPtr = VirtualAlloc(nullptr, InSize, MEM_COMMIT, PAGE_READWRITE); - - return; - } -#endif // ZEN_PLATFORM_WINDOWS - -#if ZEN_USE_MIMALLOC - void* Ptr = mi_aligned_alloc(Alignment, RoundUp(InSize, Alignment)); - m_Flags.fetch_or(kIoBufferAlloc, std::memory_order_relaxed); -#else - void* Ptr = Memory::Alloc(InSize, Alignment); -#endif - - ZEN_ASSERT(Ptr); - - m_DataPtr = Ptr; -} - -void -IoBufferCore::FreeBuffer() -{ - if (!m_DataPtr) - { - return; - } - - const uint32_t LocalFlags = m_Flags.load(std::memory_order_relaxed); -#if ZEN_PLATFORM_WINDOWS - if (LocalFlags & kLowLevelAlloc) - { - VirtualFree(const_cast<void*>(m_DataPtr), 0, MEM_DECOMMIT); - - return; - } -#endif // ZEN_PLATFORM_WINDOWS - -#if ZEN_USE_MIMALLOC - if (LocalFlags & kIoBufferAlloc) - { - return mi_free(const_cast<void*>(m_DataPtr)); - } -#endif - - ZEN_UNUSED(LocalFlags); - return Memory::Free(const_cast<void*>(m_DataPtr)); -} - -////////////////////////////////////////////////////////////////////////// - -static_assert(sizeof(IoBufferCore) == 32); - -IoBufferCore::IoBufferCore(size_t InSize) -{ - ZEN_ASSERT(InSize); - - AllocateBuffer(InSize, sizeof(void*)); - m_DataBytes = InSize; - - SetIsOwnedByThis(true); -} - -IoBufferCore::IoBufferCore(size_t InSize, size_t Alignment) -{ - ZEN_ASSERT(InSize); - - AllocateBuffer(InSize, Alignment); - m_DataBytes = InSize; - - SetIsOwnedByThis(true); -} - -IoBufferCore::~IoBufferCore() -{ - if (IsOwnedByThis() && m_DataPtr) - { - FreeBuffer(); - m_DataPtr = nullptr; - } -} - -void -IoBufferCore::DeleteThis() const -{ - // We do this just to avoid paying for the cost of a vtable - if (const IoBufferExtendedCore* _ = ExtendedCore()) - { - delete _; - } - else - { - delete this; - } -} - -void -IoBufferCore::Materialize() const -{ - if (const IoBufferExtendedCore* _ = ExtendedCore()) - { - _->Materialize(); - } -} - -void -IoBufferCore::MakeOwned(bool Immutable) -{ - if (!IsOwned()) - { - const void* OldDataPtr = m_DataPtr; - AllocateBuffer(m_DataBytes, sizeof(void*)); - memcpy(const_cast<void*>(m_DataPtr), OldDataPtr, m_DataBytes); - SetIsOwnedByThis(true); - } - - SetIsImmutable(Immutable); -} - -void* -IoBufferCore::MutableDataPointer() const -{ - EnsureDataValid(); - ZEN_ASSERT(!IsImmutable()); - return const_cast<void*>(m_DataPtr); -} - -////////////////////////////////////////////////////////////////////////// - -IoBufferExtendedCore::IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership) -: IoBufferCore(nullptr, Size) -, m_FileHandle(FileHandle) -, m_FileOffset(Offset) -{ - uint32_t NewFlags = kIsOwnedByThis | kIsExtended; - - if (TransferHandleOwnership) - { - NewFlags |= kOwnsFile; - } - m_Flags.fetch_or(NewFlags, std::memory_order_relaxed); -} - -IoBufferExtendedCore::IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size) -: IoBufferCore(Outer, nullptr, Size) -, m_FileHandle(Outer->m_FileHandle) -, m_FileOffset(Outer->m_FileOffset + Offset) -{ - m_Flags.fetch_or(kIsExtended, std::memory_order_relaxed); -} - -IoBufferExtendedCore::~IoBufferExtendedCore() -{ - if (m_MappedPointer) - { -#if ZEN_PLATFORM_WINDOWS - UnmapViewOfFile(m_MappedPointer); -#else - uint64_t MapSize = ~uint64_t(uintptr_t(m_MmapHandle)); - munmap(m_MappedPointer, MapSize); -#endif - } - - const uint32_t LocalFlags = m_Flags.load(std::memory_order_relaxed); -#if ZEN_PLATFORM_WINDOWS - if (LocalFlags & kOwnsMmap) - { - CloseHandle(m_MmapHandle); - } -#endif - - if (LocalFlags & kOwnsFile) - { - if (m_DeleteOnClose) - { -#if ZEN_PLATFORM_WINDOWS - // Mark file for deletion when final handle is closed - FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; - - SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); -#else - std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle); - unlink(FilePath.c_str()); -#endif - } -#if ZEN_PLATFORM_WINDOWS - BOOL Success = CloseHandle(m_FileHandle); -#else - int Fd = int(uintptr_t(m_FileHandle)); - bool Success = (close(Fd) == 0); -#endif - if (!Success) - { - ZEN_WARN("Error reported on file handle close, reason '{}'", GetLastErrorAsString()); - } - } - - m_DataPtr = nullptr; -} - -static constexpr size_t MappingLockCount = 128; -static_assert(IsPow2(MappingLockCount), "MappingLockCount must be power of two"); - -static RwLock g_MappingLocks[MappingLockCount]; - -static RwLock& -MappingLockForInstance(const IoBufferExtendedCore* instance) -{ - intptr_t base = (intptr_t)instance; - size_t lock_index = ((base >> 5) ^ (base >> 13)) & (MappingLockCount - 1u); - return g_MappingLocks[lock_index]; -} - -void -IoBufferExtendedCore::Materialize() const -{ - // The synchronization scheme here is very primitive, if we end up with - // a lot of contention we can make it more fine-grained - - if (m_Flags.load(std::memory_order_acquire) & kIsMaterialized) - return; - - RwLock::ExclusiveLockScope _(MappingLockForInstance(this)); - - // Someone could have gotten here first - // We can use memory_order_relaxed on this load because the mutex has already provided the fence - if (m_Flags.load(std::memory_order_relaxed) & kIsMaterialized) - return; - - uint32_t NewFlags = kIsMaterialized; - - if (m_DataBytes == 0) - { - // Fake a "valid" pointer, nobody should read this as size is zero - m_DataPtr = reinterpret_cast<uint8_t*>(&m_MmapHandle); - m_Flags.fetch_or(NewFlags, std::memory_order_release); - return; - } - - const size_t DisableMMapSizeLimit = 0x1000ull; - - if (m_DataBytes < DisableMMapSizeLimit) - { - AllocateBuffer(m_DataBytes, sizeof(void*)); - NewFlags |= kIsOwnedByThis; - -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - Ovl.Offset = DWORD(m_FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(m_FileOffset >> 32); - - DWORD dwNumberOfBytesRead = 0; - BOOL Success = ::ReadFile(m_FileHandle, (void*)m_DataPtr, DWORD(m_DataBytes), &dwNumberOfBytesRead, &Ovl); - - ZEN_ASSERT(Success); - ZEN_ASSERT(dwNumberOfBytesRead == m_DataBytes); -#else - static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); - int Fd = int(uintptr_t(m_FileHandle)); - int BytesRead = pread(Fd, (void*)m_DataPtr, m_DataBytes, m_FileOffset); - bool Success = (BytesRead > 0); -#endif // ZEN_PLATFORM_WINDOWS - - m_Flags.fetch_or(NewFlags, std::memory_order_release); - return; - } - - void* NewMmapHandle; - - const uint64_t MapOffset = m_FileOffset & ~0xffffull; - const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset; - const uint64_t MapSize = m_DataBytes + MappedOffsetDisplacement; - - ZEN_ASSERT(MapSize > 0); - -#if ZEN_PLATFORM_WINDOWS - NewMmapHandle = CreateFileMapping(m_FileHandle, - /* lpFileMappingAttributes */ nullptr, - /* flProtect */ PAGE_READONLY, - /* dwMaximumSizeLow */ 0, - /* dwMaximumSizeHigh */ 0, - /* lpName */ nullptr); - - if (NewMmapHandle == nullptr) - { - int32_t Error = zen::GetLastError(); - ZEN_ERROR("CreateFileMapping failed on file '{}', {}", zen::PathFromHandle(m_FileHandle), GetSystemErrorAsString(Error)); - throw std::system_error(std::error_code(Error, std::system_category()), - fmt::format("CreateFileMapping failed on file '{}'", zen::PathFromHandle(m_FileHandle))); - } - - NewFlags |= kOwnsMmap; - - void* MappedBase = MapViewOfFile(NewMmapHandle, - /* dwDesiredAccess */ FILE_MAP_READ, - /* FileOffsetHigh */ uint32_t(MapOffset >> 32), - /* FileOffsetLow */ uint32_t(MapOffset & 0xffFFffFFu), - /* dwNumberOfBytesToMap */ MapSize); -#else - NewMmapHandle = (void*)uintptr_t(~MapSize); // ~ so it's never null (assuming MapSize >= 0) - NewFlags |= kOwnsMmap; - - void* MappedBase = mmap( - /* addr */ nullptr, - /* length */ MapSize, - /* prot */ PROT_READ, - /* flags */ MAP_SHARED | MAP_NORESERVE, - /* fd */ int(uintptr_t(m_FileHandle)), - /* offset */ MapOffset); -#endif // ZEN_PLATFORM_WINDOWS - - if (MappedBase == nullptr) - { - int32_t Error = zen::GetLastError(); -#if ZEN_PLATFORM_WINDOWS - CloseHandle(NewMmapHandle); -#endif // ZEN_PLATFORM_WINDOWS - ZEN_ERROR("MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}', {}", - MapOffset, - MapSize, - zen::PathFromHandle(m_FileHandle), - GetSystemErrorAsString(Error)); - throw std::system_error(std::error_code(Error, std::system_category()), - fmt::format("MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}'", - MapOffset, - MapSize, - zen::PathFromHandle(m_FileHandle))); - } - - m_MappedPointer = MappedBase; - m_DataPtr = reinterpret_cast<uint8_t*>(MappedBase) + MappedOffsetDisplacement; - m_MmapHandle = NewMmapHandle; - - m_Flags.fetch_or(NewFlags, std::memory_order_release); -} - -bool -IoBufferExtendedCore::GetFileReference(IoBufferFileReference& OutRef) const -{ - if (m_FileHandle == nullptr) - { - return false; - } - - OutRef.FileHandle = m_FileHandle; - OutRef.FileChunkOffset = m_FileOffset; - OutRef.FileChunkSize = m_DataBytes; - - return true; -} - -void -IoBufferExtendedCore::MarkAsDeleteOnClose() -{ - m_DeleteOnClose = true; -} - -////////////////////////////////////////////////////////////////////////// - -IoBuffer::IoBuffer(size_t InSize) : m_Core(new IoBufferCore(InSize)) -{ - m_Core->SetIsImmutable(false); -} - -IoBuffer::IoBuffer(size_t InSize, uint64_t InAlignment) : m_Core(new IoBufferCore(InSize, InAlignment)) -{ - m_Core->SetIsImmutable(false); -} - -IoBuffer::IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t Size) -{ - if (Size == ~(0ull)) - { - Size = std::clamp<size_t>(Size, 0, OuterBuffer.Size() - Offset); - } - - ZEN_ASSERT(Offset <= OuterBuffer.Size()); - ZEN_ASSERT((Offset + Size) <= OuterBuffer.Size()); - - if (IoBufferExtendedCore* Extended = OuterBuffer.m_Core->ExtendedCore()) - { - m_Core = new IoBufferExtendedCore(Extended, Offset, Size); - } - else - { - m_Core = new IoBufferCore(OuterBuffer.m_Core, reinterpret_cast<const uint8_t*>(OuterBuffer.Data()) + Offset, Size); - } -} - -IoBuffer::IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize) -: m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ true)) -{ -} - -IoBuffer::IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize) -: m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ false)) -{ -} - -bool -IoBuffer::GetFileReference(IoBufferFileReference& OutRef) const -{ - if (IoBufferExtendedCore* ExtCore = m_Core->ExtendedCore()) - { - if (ExtCore->GetFileReference(OutRef)) - { - return true; - } - } - - // Not a file reference - - OutRef.FileHandle = 0; - OutRef.FileChunkOffset = ~0ull; - OutRef.FileChunkSize = 0; - - return false; -} - -void -IoBuffer::MarkAsDeleteOnClose() -{ - if (IoBufferExtendedCore* ExtCore = m_Core->ExtendedCore()) - { - ExtCore->MarkAsDeleteOnClose(); - } -} - -////////////////////////////////////////////////////////////////////////// - -IoBuffer -IoBufferBuilder::ReadFromFileMaybe(IoBuffer& InBuffer) -{ - IoBufferFileReference FileRef; - if (InBuffer.GetFileReference(/* out */ FileRef)) - { - IoBuffer OutBuffer(FileRef.FileChunkSize); - -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - const uint64_t NumberOfBytesToRead = FileRef.FileChunkSize; - const uint64_t& FileOffset = FileRef.FileChunkOffset; - - Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(FileOffset >> 32); - - DWORD dwNumberOfBytesRead = 0; - BOOL Success = ::ReadFile(FileRef.FileHandle, OutBuffer.MutableData(), DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); -#else - int Fd = int(intptr_t(FileRef.FileHandle)); - int Result = pread(Fd, OutBuffer.MutableData(), size_t(FileRef.FileChunkSize), off_t(FileRef.FileChunkOffset)); - bool Success = (Result < 0); - - uint32_t dwNumberOfBytesRead = uint32_t(Result); -#endif - - if (!Success) - { - ThrowLastError("ReadFile failed in IoBufferBuilder::ReadFromFileMaybe"); - } - - ZEN_ASSERT(dwNumberOfBytesRead == FileRef.FileChunkSize); - - return OutBuffer; - } - else - { - return InBuffer; - } -} - -IoBuffer -IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size) -{ - return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); -} - -IoBuffer -IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) -{ - uint64_t FileSize; - -#if ZEN_PLATFORM_WINDOWS - CAtlFile DataFile; - - DWORD ShareOptions = FILE_SHARE_DELETE | FILE_SHARE_WRITE | FILE_SHARE_DELETE | FILE_SHARE_READ; - HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, ShareOptions, OPEN_EXISTING); - - if (FAILED(hRes)) - { - return {}; - } - - DataFile.GetSize((ULONGLONG&)FileSize); -#else - int Flags = O_RDONLY | O_CLOEXEC; - int Fd = open(FileName.c_str(), Flags); - if (Fd < 0) - { - return {}; - } - - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - fstat(Fd, &Stat); - FileSize = Stat.st_size; -#endif // ZEN_PLATFORM_WINDOWS - - // TODO: should validate that offset is in range - - if (Size == ~0ull) - { - Size = FileSize - Offset; - } - else - { - // Clamp size - if ((Offset + Size) > FileSize) - { - Size = FileSize - Offset; - } - } - - if (Size) - { -#if ZEN_PLATFORM_WINDOWS - void* Fd = DataFile.Detach(); -#endif - IoBuffer Iob(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size); - Iob.m_Core->SetIsWholeFile(Offset == 0 && Size == FileSize); - return Iob; - } - -#if !ZEN_PLATFORM_WINDOWS - close(Fd); -#endif - - // For an empty file, we may as well just return an empty memory IoBuffer - return IoBuffer(IoBuffer::Wrap, "", 0); -} - -IoBuffer -IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) -{ - uint64_t FileSize; - void* Handle; - -#if ZEN_PLATFORM_WINDOWS - CAtlFile DataFile; - - // We need to open with DELETE since this is used for the case - // when a file has been written to a staging directory, and is going - // to be moved in place - - HRESULT hRes = DataFile.Create(FileName.native().c_str(), GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING); - - if (FAILED(hRes)) - { - return {}; - } - - DataFile.GetSize((ULONGLONG&)FileSize); - - Handle = DataFile.Detach(); -#else - int Fd = open(FileName.native().c_str(), O_RDONLY); - if (Fd < 0) - { - return {}; - } - - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - fstat(Fd, &Stat); - FileSize = Stat.st_size; - - Handle = (void*)uintptr_t(Fd); -#endif // ZEN_PLATFORM_WINDOWS - - IoBuffer Iob(IoBuffer::File, Handle, 0, FileSize); - Iob.m_Core->SetIsWholeFile(true); - - return Iob; -} - -IoHash -HashBuffer(IoBuffer& Buffer) -{ - // TODO: handle disk buffers with special path - return IoHash::HashBuffer(Buffer.Data(), Buffer.Size()); -} - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_WITH_TESTS - -void -iobuffer_forcelink() -{ -} - -TEST_CASE("IoBuffer") -{ - zen::IoBuffer buffer1; - zen::IoBuffer buffer2(16384); - zen::IoBuffer buffer3(buffer2, 0, buffer2.Size()); -} - -#endif - -} // namespace zen diff --git a/zencore/iohash.cpp b/zencore/iohash.cpp deleted file mode 100644 index 77076c133..000000000 --- a/zencore/iohash.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/iohash.h> - -#include <zencore/blake3.h> -#include <zencore/compositebuffer.h> -#include <zencore/string.h> -#include <zencore/testing.h> - -#include <gsl/gsl-lite.hpp> - -namespace zen { - -const IoHash IoHash::Zero{}; // Initialized to all zeros - -IoHash -IoHash::HashBuffer(const void* data, size_t byteCount) -{ - BLAKE3 b3 = BLAKE3::HashMemory(data, byteCount); - - IoHash io; - memcpy(io.Hash, b3.Hash, sizeof io.Hash); - - return io; -} - -IoHash -IoHash::HashBuffer(const CompositeBuffer& Buffer) -{ - IoHashStream Hasher; - - for (const SharedBuffer& Segment : Buffer.GetSegments()) - { - Hasher.Append(Segment.GetData(), Segment.GetSize()); - } - - return Hasher.GetHash(); -} - -IoHash -IoHash::FromHexString(const char* string) -{ - return FromHexString({string, sizeof(IoHash::Hash) * 2}); -} - -IoHash -IoHash::FromHexString(std::string_view string) -{ - ZEN_ASSERT(string.size() == 2 * sizeof(IoHash::Hash)); - - IoHash io; - - ParseHexBytes(string.data(), string.size(), io.Hash); - - return io; -} - -const char* -IoHash::ToHexString(char* outString /* 40 characters + NUL terminator */) const -{ - ToHexBytes(Hash, sizeof(IoHash), outString); - outString[2 * sizeof(IoHash)] = '\0'; - - return outString; -} - -StringBuilderBase& -IoHash::ToHexString(StringBuilderBase& outBuilder) const -{ - String_t Str; - ToHexString(Str); - - outBuilder.AppendRange(Str, &Str[StringLength]); - - return outBuilder; -} - -std::string -IoHash::ToHexString() const -{ - String_t Str; - ToHexString(Str); - - return Str; -} - -} // namespace zen diff --git a/zencore/logging.cpp b/zencore/logging.cpp deleted file mode 100644 index a6423e2dc..000000000 --- a/zencore/logging.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/logging.h" - -#include <spdlog/sinks/stdout_color_sinks.h> - -namespace zen { - -// We shadow the underlying spdlog default logger, in order to avoid a bunch of overhead -spdlog::logger* TheDefaultLogger; - -} // namespace zen - -namespace zen::logging { - -spdlog::logger& -Default() -{ - return *TheDefaultLogger; -} - -void -SetDefault(std::shared_ptr<spdlog::logger> NewDefaultLogger) -{ - spdlog::set_default_logger(NewDefaultLogger); - TheDefaultLogger = spdlog::default_logger_raw(); -} - -spdlog::logger& -Get(std::string_view Name) -{ - std::shared_ptr<spdlog::logger> Logger = spdlog::get(std::string(Name)); - - if (!Logger) - { - Logger = Default().clone(std::string(Name)); - spdlog::register_logger(Logger); - } - - return *Logger; -} - -std::once_flag ConsoleInitFlag; -std::shared_ptr<spdlog::logger> ConLogger; - -spdlog::logger& -ConsoleLog() -{ - std::call_once(ConsoleInitFlag, [&] { - ConLogger = spdlog::stdout_color_mt("console"); - - ConLogger->set_pattern("%v"); - }); - - return *ConLogger; -} - -std::shared_ptr<spdlog::logger> TheErrorLogger; - -spdlog::logger* -ErrorLog() -{ - return TheErrorLogger.get(); -} - -void -SetErrorLog(std::shared_ptr<spdlog::logger>&& NewErrorLogger) -{ - TheErrorLogger = std::move(NewErrorLogger); -} - -void -InitializeLogging() -{ - TheDefaultLogger = spdlog::default_logger_raw(); -} - -void -ShutdownLogging() -{ - spdlog::drop_all(); - spdlog::shutdown(); -} - -} // namespace zen::logging diff --git a/zencore/md5.cpp b/zencore/md5.cpp deleted file mode 100644 index 4ec145697..000000000 --- a/zencore/md5.cpp +++ /dev/null @@ -1,463 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/md5.h> -#include <zencore/string.h> -#include <zencore/testing.h> -#include <zencore/zencore.h> - -#include <string.h> -#include <string_view> - -/* - ********************************************************************** - ** md5.h -- Header file for implementation of MD5 ** - ** RSA Data Security, Inc. MD5 Message Digest Algorithm ** - ** Created: 2/17/90 RLR ** - ** Revised: 12/27/90 SRD,AJ,BSK,JT Reference C version ** - ** Revised (for MD5): RLR 4/27/91 ** - ** -- G modified to have y&~z instead of y&z ** - ** -- FF, GG, HH modified to add in last register done ** - ** -- Access pattern: round 2 works mod 5, round 3 works mod 3 ** - ** -- distinct additive constant for each step ** - ** -- round 4 added, working mod 7 ** - ********************************************************************** - */ - -/* - ********************************************************************** - ** Copyright (C) 1990, RSA Data Security, Inc. All rights reserved. ** - ** ** - ** License to copy and use this software is granted provided that ** - ** it is identified as the "RSA Data Security, Inc. MD5 Message ** - ** Digest Algorithm" in all material mentioning or referencing this ** - ** software or this function. ** - ** ** - ** License is also granted to make and use derivative works ** - ** provided that such works are identified as "derived from the RSA ** - ** Data Security, Inc. MD5 Message Digest Algorithm" in all ** - ** material mentioning or referencing the derived work. ** - ** ** - ** RSA Data Security, Inc. makes no representations concerning ** - ** either the merchantability of this software or the suitability ** - ** of this software for any particular purpose. It is provided "as ** - ** is" without express or implied warranty of any kind. ** - ** ** - ** These notices must be retained in any copies of any part of this ** - ** documentation and/or software. ** - ********************************************************************** - */ - -/* Data structure for MD5 (Message Digest) computation */ -struct MD5_CTX -{ - uint32_t i[2]; /* number of _bits_ handled mod 2^64 */ - uint32_t buf[4]; /* scratch buffer */ - unsigned char in[64]; /* input buffer */ - unsigned char digest[16]; /* actual digest after MD5Final call */ -}; - -void MD5Init(); -void MD5Update(); -void MD5Final(); - -/* - ********************************************************************** - ** End of md5.h ** - ******************************* (cut) ******************************** - */ - -/* - ********************************************************************** - ** md5.c ** - ** RSA Data Security, Inc. MD5 Message Digest Algorithm ** - ** Created: 2/17/90 RLR ** - ** Revised: 1/91 SRD,AJ,BSK,JT Reference C Version ** - ********************************************************************** - */ - -/* - ********************************************************************** - ** Copyright (C) 1990, RSA Data Security, Inc. All rights reserved. ** - ** ** - ** License to copy and use this software is granted provided that ** - ** it is identified as the "RSA Data Security, Inc. MD5 Message ** - ** Digest Algorithm" in all material mentioning or referencing this ** - ** software or this function. ** - ** ** - ** License is also granted to make and use derivative works ** - ** provided that such works are identified as "derived from the RSA ** - ** Data Security, Inc. MD5 Message Digest Algorithm" in all ** - ** material mentioning or referencing the derived work. ** - ** ** - ** RSA Data Security, Inc. makes no representations concerning ** - ** either the merchantability of this software or the suitability ** - ** of this software for any particular purpose. It is provided "as ** - ** is" without express or implied warranty of any kind. ** - ** ** - ** These notices must be retained in any copies of any part of this ** - ** documentation and/or software. ** - ********************************************************************** - */ - -/* -- include the following line if the md5.h header file is separate -- */ -/* #include "md5.h" */ - -/* forward declaration */ -static void Transform(uint32_t* buf, uint32_t* in); - -static unsigned char PADDING[64] = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; - -/* F, G and H are basic MD5 functions: selection, majority, parity */ -#define F(x, y, z) (((x) & (y)) | ((~x) & (z))) -#define G(x, y, z) (((x) & (z)) | ((y) & (~z))) -#define H(x, y, z) ((x) ^ (y) ^ (z)) -#define I(x, y, z) ((y) ^ ((x) | (~z))) - -/* ROTATE_LEFT rotates x left n bits */ -#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) - -/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4 */ -/* Rotation is separate from addition to prevent recomputation */ -#define FF(a, b, c, d, x, s, ac) \ - { \ - (a) += F((b), (c), (d)) + (x) + (uint32_t)(ac); \ - (a) = ROTATE_LEFT((a), (s)); \ - (a) += (b); \ - } -#define GG(a, b, c, d, x, s, ac) \ - { \ - (a) += G((b), (c), (d)) + (x) + (uint32_t)(ac); \ - (a) = ROTATE_LEFT((a), (s)); \ - (a) += (b); \ - } -#define HH(a, b, c, d, x, s, ac) \ - { \ - (a) += H((b), (c), (d)) + (x) + (uint32_t)(ac); \ - (a) = ROTATE_LEFT((a), (s)); \ - (a) += (b); \ - } -#define II(a, b, c, d, x, s, ac) \ - { \ - (a) += I((b), (c), (d)) + (x) + (uint32_t)(ac); \ - (a) = ROTATE_LEFT((a), (s)); \ - (a) += (b); \ - } - -void -MD5Init(MD5_CTX* mdContext) -{ - mdContext->i[0] = mdContext->i[1] = (uint32_t)0; - - /* Load magic initialization constants. - */ - mdContext->buf[0] = (uint32_t)0x67452301; - mdContext->buf[1] = (uint32_t)0xefcdab89; - mdContext->buf[2] = (uint32_t)0x98badcfe; - mdContext->buf[3] = (uint32_t)0x10325476; -} - -void -MD5Update(MD5_CTX* mdContext, unsigned char* inBuf, unsigned int inLen) -{ - uint32_t in[16]; - int mdi; - unsigned int i, ii; - - /* compute number of bytes mod 64 */ - mdi = (int)((mdContext->i[0] >> 3) & 0x3F); - - /* update number of bits */ - if ((mdContext->i[0] + ((uint32_t)inLen << 3)) < mdContext->i[0]) - mdContext->i[1]++; - mdContext->i[0] += ((uint32_t)inLen << 3); - mdContext->i[1] += ((uint32_t)inLen >> 29); - - while (inLen--) - { - /* add new character to buffer, increment mdi */ - mdContext->in[mdi++] = *inBuf++; - - /* transform if necessary */ - if (mdi == 0x40) - { - for (i = 0, ii = 0; i < 16; i++, ii += 4) - in[i] = (((uint32_t)mdContext->in[ii + 3]) << 24) | (((uint32_t)mdContext->in[ii + 2]) << 16) | - (((uint32_t)mdContext->in[ii + 1]) << 8) | ((uint32_t)mdContext->in[ii]); - Transform(mdContext->buf, in); - mdi = 0; - } - } -} - -void -MD5Final(MD5_CTX* mdContext) -{ - uint32_t in[16]; - int mdi; - unsigned int i, ii; - unsigned int padLen; - - /* save number of bits */ - in[14] = mdContext->i[0]; - in[15] = mdContext->i[1]; - - /* compute number of bytes mod 64 */ - mdi = (int)((mdContext->i[0] >> 3) & 0x3F); - - /* pad out to 56 mod 64 */ - padLen = (mdi < 56) ? (56 - mdi) : (120 - mdi); - MD5Update(mdContext, PADDING, padLen); - - /* append length in bits and transform */ - for (i = 0, ii = 0; i < 14; i++, ii += 4) - in[i] = (((uint32_t)mdContext->in[ii + 3]) << 24) | (((uint32_t)mdContext->in[ii + 2]) << 16) | - (((uint32_t)mdContext->in[ii + 1]) << 8) | ((uint32_t)mdContext->in[ii]); - Transform(mdContext->buf, in); - - /* store buffer in digest */ - for (i = 0, ii = 0; i < 4; i++, ii += 4) - { - mdContext->digest[ii] = (unsigned char)(mdContext->buf[i] & 0xFF); - mdContext->digest[ii + 1] = (unsigned char)((mdContext->buf[i] >> 8) & 0xFF); - mdContext->digest[ii + 2] = (unsigned char)((mdContext->buf[i] >> 16) & 0xFF); - mdContext->digest[ii + 3] = (unsigned char)((mdContext->buf[i] >> 24) & 0xFF); - } -} - -/* Basic MD5 step. Transform buf based on in. - */ -static void -Transform(uint32_t* buf, uint32_t* in) -{ - uint32_t a = buf[0], b = buf[1], c = buf[2], d = buf[3]; - - /* Round 1 */ -#define S11 7 -#define S12 12 -#define S13 17 -#define S14 22 - FF(a, b, c, d, in[0], S11, 3614090360); /* 1 */ - FF(d, a, b, c, in[1], S12, 3905402710); /* 2 */ - FF(c, d, a, b, in[2], S13, 606105819); /* 3 */ - FF(b, c, d, a, in[3], S14, 3250441966); /* 4 */ - FF(a, b, c, d, in[4], S11, 4118548399); /* 5 */ - FF(d, a, b, c, in[5], S12, 1200080426); /* 6 */ - FF(c, d, a, b, in[6], S13, 2821735955); /* 7 */ - FF(b, c, d, a, in[7], S14, 4249261313); /* 8 */ - FF(a, b, c, d, in[8], S11, 1770035416); /* 9 */ - FF(d, a, b, c, in[9], S12, 2336552879); /* 10 */ - FF(c, d, a, b, in[10], S13, 4294925233); /* 11 */ - FF(b, c, d, a, in[11], S14, 2304563134); /* 12 */ - FF(a, b, c, d, in[12], S11, 1804603682); /* 13 */ - FF(d, a, b, c, in[13], S12, 4254626195); /* 14 */ - FF(c, d, a, b, in[14], S13, 2792965006); /* 15 */ - FF(b, c, d, a, in[15], S14, 1236535329); /* 16 */ - - /* Round 2 */ -#define S21 5 -#define S22 9 -#define S23 14 -#define S24 20 - GG(a, b, c, d, in[1], S21, 4129170786); /* 17 */ - GG(d, a, b, c, in[6], S22, 3225465664); /* 18 */ - GG(c, d, a, b, in[11], S23, 643717713); /* 19 */ - GG(b, c, d, a, in[0], S24, 3921069994); /* 20 */ - GG(a, b, c, d, in[5], S21, 3593408605); /* 21 */ - GG(d, a, b, c, in[10], S22, 38016083); /* 22 */ - GG(c, d, a, b, in[15], S23, 3634488961); /* 23 */ - GG(b, c, d, a, in[4], S24, 3889429448); /* 24 */ - GG(a, b, c, d, in[9], S21, 568446438); /* 25 */ - GG(d, a, b, c, in[14], S22, 3275163606); /* 26 */ - GG(c, d, a, b, in[3], S23, 4107603335); /* 27 */ - GG(b, c, d, a, in[8], S24, 1163531501); /* 28 */ - GG(a, b, c, d, in[13], S21, 2850285829); /* 29 */ - GG(d, a, b, c, in[2], S22, 4243563512); /* 30 */ - GG(c, d, a, b, in[7], S23, 1735328473); /* 31 */ - GG(b, c, d, a, in[12], S24, 2368359562); /* 32 */ - - /* Round 3 */ -#define S31 4 -#define S32 11 -#define S33 16 -#define S34 23 - HH(a, b, c, d, in[5], S31, 4294588738); /* 33 */ - HH(d, a, b, c, in[8], S32, 2272392833); /* 34 */ - HH(c, d, a, b, in[11], S33, 1839030562); /* 35 */ - HH(b, c, d, a, in[14], S34, 4259657740); /* 36 */ - HH(a, b, c, d, in[1], S31, 2763975236); /* 37 */ - HH(d, a, b, c, in[4], S32, 1272893353); /* 38 */ - HH(c, d, a, b, in[7], S33, 4139469664); /* 39 */ - HH(b, c, d, a, in[10], S34, 3200236656); /* 40 */ - HH(a, b, c, d, in[13], S31, 681279174); /* 41 */ - HH(d, a, b, c, in[0], S32, 3936430074); /* 42 */ - HH(c, d, a, b, in[3], S33, 3572445317); /* 43 */ - HH(b, c, d, a, in[6], S34, 76029189); /* 44 */ - HH(a, b, c, d, in[9], S31, 3654602809); /* 45 */ - HH(d, a, b, c, in[12], S32, 3873151461); /* 46 */ - HH(c, d, a, b, in[15], S33, 530742520); /* 47 */ - HH(b, c, d, a, in[2], S34, 3299628645); /* 48 */ - - /* Round 4 */ -#define S41 6 -#define S42 10 -#define S43 15 -#define S44 21 - II(a, b, c, d, in[0], S41, 4096336452); /* 49 */ - II(d, a, b, c, in[7], S42, 1126891415); /* 50 */ - II(c, d, a, b, in[14], S43, 2878612391); /* 51 */ - II(b, c, d, a, in[5], S44, 4237533241); /* 52 */ - II(a, b, c, d, in[12], S41, 1700485571); /* 53 */ - II(d, a, b, c, in[3], S42, 2399980690); /* 54 */ - II(c, d, a, b, in[10], S43, 4293915773); /* 55 */ - II(b, c, d, a, in[1], S44, 2240044497); /* 56 */ - II(a, b, c, d, in[8], S41, 1873313359); /* 57 */ - II(d, a, b, c, in[15], S42, 4264355552); /* 58 */ - II(c, d, a, b, in[6], S43, 2734768916); /* 59 */ - II(b, c, d, a, in[13], S44, 1309151649); /* 60 */ - II(a, b, c, d, in[4], S41, 4149444226); /* 61 */ - II(d, a, b, c, in[11], S42, 3174756917); /* 62 */ - II(c, d, a, b, in[2], S43, 718787259); /* 63 */ - II(b, c, d, a, in[9], S44, 3951481745); /* 64 */ - - buf[0] += a; - buf[1] += b; - buf[2] += c; - buf[3] += d; -} - -/* - ********************************************************************** - ** End of md5.c ** - ******************************* (cut) ******************************** - */ - -#undef FF -#undef GG -#undef HH -#undef II -#undef F -#undef G -#undef H -#undef I - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -MD5 MD5::Zero; // Initialized to all zeroes - -////////////////////////////////////////////////////////////////////////// - -MD5Stream::MD5Stream() -{ - Reset(); -} - -void -MD5Stream::Reset() -{ -} - -MD5Stream& -MD5Stream::Append(const void* Data, size_t ByteCount) -{ - ZEN_UNUSED(Data); - ZEN_UNUSED(ByteCount); - - return *this; -} - -MD5 -MD5Stream::GetHash() -{ - MD5 md5{}; - - return md5; -} - -////////////////////////////////////////////////////////////////////////// - -MD5 -MD5::HashMemory(const void* data, size_t byteCount) -{ - return MD5Stream().Append(data, byteCount).GetHash(); -} - -MD5 -MD5::FromHexString(const char* string) -{ - MD5 md5; - - ParseHexBytes(string, 40, md5.Hash); - - return md5; -} - -const char* -MD5::ToHexString(char* outString /* 32 characters + NUL terminator */) const -{ - ToHexBytes(Hash, sizeof(MD5), outString); - outString[2 * sizeof(MD5)] = '\0'; - - return outString; -} - -StringBuilderBase& -MD5::ToHexString(StringBuilderBase& outBuilder) const -{ - char str[41]; - ToHexString(str); - - outBuilder.AppendRange(str, &str[40]); - - return outBuilder; -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -void -md5_forcelink() -{ -} - -// doctest::String -// toString(const MD5& value) -// { -// char md5text[2 * sizeof(MD5) + 1]; -// value.ToHexString(md5text); - -// return md5text; -// } - -TEST_CASE("MD5") -{ - using namespace std::literals; - - auto Input = "jumblesmcgee"sv; - auto Output = "28f2200a59c60b75947099d750c2cc50"sv; - - MD5Stream Stream; - Stream.Append(Input.data(), Input.length()); - MD5 Result = Stream.GetHash(); - - MD5::String_t Buffer; - Result.ToHexString(Buffer); - - CHECK(Output.compare(Buffer)); - - MD5 Reresult = MD5::FromHexString(Buffer); - Reresult.ToHexString(Buffer); - CHECK(Output.compare(Buffer)); -} - -#endif - -} // namespace zen diff --git a/zencore/memory.cpp b/zencore/memory.cpp deleted file mode 100644 index 1f148cede..000000000 --- a/zencore/memory.cpp +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/intmath.h> -#include <zencore/memory.h> -#include <zencore/testing.h> -#include <zencore/zencore.h> - -#if ZEN_PLATFORM_WINDOWS -# include <malloc.h> -ZEN_THIRD_PARTY_INCLUDES_START -# include <mimalloc.h> -ZEN_THIRD_PARTY_INCLUDES_END -#else -# include <cstdlib> -#endif - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -static void* -AlignedAllocImpl(size_t Size, size_t Alignment) -{ -#if ZEN_PLATFORM_WINDOWS -# if ZEN_USE_MIMALLOC && 0 /* this path is not functional */ - return mi_aligned_alloc(Alignment, Size); -# else - return _aligned_malloc(Size, Alignment); -# endif -#else - // aligned_alloc() states that size must be a multiple of alignment. Some - // platforms return null if this requirement isn't met. - Size = (Size + Alignment - 1) & ~(Alignment - 1); - return std::aligned_alloc(Alignment, Size); -#endif -} - -void -AlignedFreeImpl(void* ptr) -{ - if (ptr == nullptr) - return; - -#if ZEN_PLATFORM_WINDOWS -# if ZEN_USE_MIMALLOC && 0 /* this path is not functional */ - return mi_free(ptr); -# else - _aligned_free(ptr); -# endif -#else - std::free(ptr); -#endif -} - -////////////////////////////////////////////////////////////////////////// - -MemoryArena::MemoryArena() -{ -} - -MemoryArena::~MemoryArena() -{ -} - -void* -MemoryArena::Alloc(size_t Size, size_t Alignment) -{ - return AlignedAllocImpl(Size, Alignment); -} - -void -MemoryArena::Free(void* ptr) -{ - AlignedFreeImpl(ptr); -} - -////////////////////////////////////////////////////////////////////////// - -void* -Memory::Alloc(size_t Size, size_t Alignment) -{ - return AlignedAllocImpl(Size, Alignment); -} - -void -Memory::Free(void* ptr) -{ - AlignedFreeImpl(ptr); -} - -////////////////////////////////////////////////////////////////////////// - -ChunkingLinearAllocator::ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment) -: m_ChunkSize(ChunkSize) -, m_ChunkAlignment(ChunkAlignment) -{ -} - -ChunkingLinearAllocator::~ChunkingLinearAllocator() -{ - Reset(); -} - -void -ChunkingLinearAllocator::Reset() -{ - for (void* ChunkEntry : m_ChunkList) - { - Memory::Free(ChunkEntry); - } - m_ChunkList.clear(); - - m_ChunkCursor = nullptr; - m_ChunkBytesRemain = 0; -} - -void* -ChunkingLinearAllocator::Alloc(size_t Size, size_t Alignment) -{ - ZEN_ASSERT_SLOW(zen::IsPow2(Alignment)); - - // This could be improved in a bunch of ways - // - // * We pessimistically allocate memory even though there may be enough memory available for a single allocation due to the way we take - // alignment into account below - // * The block allocation size could be chosen to minimize slack for the case when multiple oversize allocations are made rather than - // minimizing the number of chunks - // * ... - - const uint64_t AllocationSize = zen::RoundUp(Size, Alignment); - - if (m_ChunkBytesRemain < (AllocationSize + Alignment - 1)) - { - const uint64_t ChunkSize = zen::RoundUp(zen::Max(m_ChunkSize, Size), m_ChunkSize); - void* ChunkPtr = Memory::Alloc(ChunkSize, m_ChunkAlignment); - m_ChunkCursor = reinterpret_cast<uint8_t*>(ChunkPtr); - m_ChunkBytesRemain = ChunkSize; - m_ChunkList.push_back(ChunkPtr); - } - - const uint64_t AlignFixup = (Alignment - reinterpret_cast<uintptr_t>(m_ChunkCursor)) & (Alignment - 1); - void* ReturnPtr = m_ChunkCursor + AlignFixup; - const uint64_t Delta = AlignFixup + AllocationSize; - - ZEN_ASSERT_SLOW(m_ChunkBytesRemain >= Delta); - - m_ChunkCursor += Delta; - m_ChunkBytesRemain -= Delta; - - ZEN_ASSERT_SLOW(IsPointerAligned(ReturnPtr, Alignment)); - - return ReturnPtr; -} - -////////////////////////////////////////////////////////////////////////// -// -// Unit tests -// - -#if ZEN_WITH_TESTS - -TEST_CASE("ChunkingLinearAllocator") -{ - ChunkingLinearAllocator Allocator(4096); - - void* p1 = Allocator.Alloc(1, 1); - void* p2 = Allocator.Alloc(1, 1); - - CHECK(p1 != p2); - - void* p3 = Allocator.Alloc(1, 4); - CHECK(IsPointerAligned(p3, 4)); - - void* p3_2 = Allocator.Alloc(1, 4); - CHECK(IsPointerAligned(p3_2, 4)); - - void* p4 = Allocator.Alloc(1, 8); - CHECK(IsPointerAligned(p4, 8)); - - for (int i = 0; i < 100; ++i) - { - void* p0 = Allocator.Alloc(64); - ZEN_UNUSED(p0); - } -} - -TEST_CASE("MemoryView") -{ - { - uint8_t Array1[16] = {}; - MemoryView View1 = MakeMemoryView(Array1); - CHECK(View1.GetSize() == 16); - } - - { - uint32_t Array2[16] = {}; - MemoryView View2 = MakeMemoryView(Array2); - CHECK(View2.GetSize() == 64); - } - - CHECK(MakeMemoryView<float>({1.0f, 1.2f}).GetSize() == 8); -} - -void -memory_forcelink() -{ -} - -#endif - -} // namespace zen diff --git a/zencore/mpscqueue.cpp b/zencore/mpscqueue.cpp deleted file mode 100644 index 29c76c3ca..000000000 --- a/zencore/mpscqueue.cpp +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/mpscqueue.h> - -#include <zencore/testing.h> -#include <string> - -namespace zen { - -#if ZEN_WITH_TESTS && 0 -TEST_CASE("mpsc") -{ - MpscQueue<std::string> Queue; - Queue.Enqueue("hello"); - std::optional<std::string> Value = Queue.Dequeue(); - CHECK_EQ(Value, "hello"); -} -#endif - -void -mpscqueue_forcelink() -{ -} - -} // namespace zen
\ No newline at end of file diff --git a/zencore/refcount.cpp b/zencore/refcount.cpp deleted file mode 100644 index c6c47b04d..000000000 --- a/zencore/refcount.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/refcount.h> - -#include <zencore/testing.h> - -#include <functional> - -namespace zen { - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -struct TestRefClass : public RefCounted -{ - ~TestRefClass() - { - if (OnDestroy) - OnDestroy(); - } - - using RefCounted::RefCount; - - std::function<void()> OnDestroy; -}; - -void -refcount_forcelink() -{ -} - -TEST_CASE("RefPtr") -{ - RefPtr<TestRefClass> Ref; - Ref = new TestRefClass; - - bool IsDestroyed = false; - Ref->OnDestroy = [&] { IsDestroyed = true; }; - - CHECK(IsDestroyed == false); - CHECK(Ref->RefCount() == 1); - - RefPtr<TestRefClass> Ref2; - Ref2 = Ref; - - CHECK(IsDestroyed == false); - CHECK(Ref->RefCount() == 2); - - RefPtr<TestRefClass> Ref3; - Ref2 = Ref3; - - CHECK(IsDestroyed == false); - CHECK(Ref->RefCount() == 1); - Ref = Ref3; - - CHECK(IsDestroyed == true); -} - -#endif - -} // namespace zen diff --git a/zencore/session.cpp b/zencore/session.cpp deleted file mode 100644 index ce4bfae1b..000000000 --- a/zencore/session.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/session.h" - -#include <zencore/uid.h> - -#include <mutex> - -namespace zen { - -static Oid GlobalSessionId; -static char GlobalSessionString[Oid::StringLength]; -static std::once_flag SessionInitFlag; - -Oid -GetSessionId() -{ - std::call_once(SessionInitFlag, [&] { - GlobalSessionId.Generate(); - GlobalSessionId.ToString(GlobalSessionString); - }); - - return GlobalSessionId; -} - -std::string_view -GetSessionIdString() -{ - // Ensure we actually have a generated session identifier - std::ignore = GetSessionId(); - - return std::string_view(GlobalSessionString, Oid::StringLength); -} - -} // namespace zen
\ No newline at end of file diff --git a/zencore/sha1.cpp b/zencore/sha1.cpp deleted file mode 100644 index 3ee74d7d8..000000000 --- a/zencore/sha1.cpp +++ /dev/null @@ -1,443 +0,0 @@ -// ////////////////////////////////////////////////////////// -// sha1.cpp -// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. -// see http://create.stephan-brumme.com/disclaimer.html -// - -#include <zencore/sha1.h> -#include <zencore/string.h> -#include <zencore/testing.h> -#include <zencore/zencore.h> - -#include <string.h> - -// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN -#if ZEN_PLATFORM_LINUX -# include <endian.h> -#endif - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -SHA1 SHA1::Zero; // Initialized to all zeroes - -////////////////////////////////////////////////////////////////////////// - -SHA1Stream::SHA1Stream() -{ - Reset(); -} - -void -SHA1Stream::Reset() -{ - m_NumBytes = 0; - m_BufferSize = 0; - - // according to RFC 1321 - m_Hash[0] = 0x67452301; - m_Hash[1] = 0xefcdab89; - m_Hash[2] = 0x98badcfe; - m_Hash[3] = 0x10325476; - m_Hash[4] = 0xc3d2e1f0; -} - -namespace { - // mix functions for processBlock() - inline uint32_t f1(uint32_t b, uint32_t c, uint32_t d) - { - return d ^ (b & (c ^ d)); // original: f = (b & c) | ((~b) & d); - } - - inline uint32_t f2(uint32_t b, uint32_t c, uint32_t d) { return b ^ c ^ d; } - - inline uint32_t f3(uint32_t b, uint32_t c, uint32_t d) { return (b & c) | (b & d) | (c & d); } - - inline uint32_t rotate(uint32_t a, uint32_t c) { return (a << c) | (a >> (32 - c)); } - - inline uint32_t swap(uint32_t x) - { -#if defined(__GNUC__) || defined(__clang__) - return __builtin_bswap32(x); -#endif -#ifdef MSC_VER - return _byteswap_ulong(x); -#endif - - return (x >> 24) | ((x >> 8) & 0x0000FF00) | ((x << 8) & 0x00FF0000) | (x << 24); - } -} // namespace - -/// process 64 bytes -void -SHA1Stream::ProcessBlock(const void* data) -{ - // get last hash - uint32_t a = m_Hash[0]; - uint32_t b = m_Hash[1]; - uint32_t c = m_Hash[2]; - uint32_t d = m_Hash[3]; - uint32_t e = m_Hash[4]; - - // data represented as 16x 32-bit words - const uint32_t* input = (uint32_t*)data; - // convert to big endian - uint32_t words[80]; - for (int i = 0; i < 16; i++) -#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) - words[i] = input[i]; -#else - words[i] = swap(input[i]); -#endif - - // extend to 80 words - for (int i = 16; i < 80; i++) - words[i] = rotate(words[i - 3] ^ words[i - 8] ^ words[i - 14] ^ words[i - 16], 1); - - // first round - for (int i = 0; i < 4; i++) - { - int offset = 5 * i; - e += rotate(a, 5) + f1(b, c, d) + words[offset] + 0x5a827999; - b = rotate(b, 30); - d += rotate(e, 5) + f1(a, b, c) + words[offset + 1] + 0x5a827999; - a = rotate(a, 30); - c += rotate(d, 5) + f1(e, a, b) + words[offset + 2] + 0x5a827999; - e = rotate(e, 30); - b += rotate(c, 5) + f1(d, e, a) + words[offset + 3] + 0x5a827999; - d = rotate(d, 30); - a += rotate(b, 5) + f1(c, d, e) + words[offset + 4] + 0x5a827999; - c = rotate(c, 30); - } - - // second round - for (int i = 4; i < 8; i++) - { - int offset = 5 * i; - e += rotate(a, 5) + f2(b, c, d) + words[offset] + 0x6ed9eba1; - b = rotate(b, 30); - d += rotate(e, 5) + f2(a, b, c) + words[offset + 1] + 0x6ed9eba1; - a = rotate(a, 30); - c += rotate(d, 5) + f2(e, a, b) + words[offset + 2] + 0x6ed9eba1; - e = rotate(e, 30); - b += rotate(c, 5) + f2(d, e, a) + words[offset + 3] + 0x6ed9eba1; - d = rotate(d, 30); - a += rotate(b, 5) + f2(c, d, e) + words[offset + 4] + 0x6ed9eba1; - c = rotate(c, 30); - } - - // third round - for (int i = 8; i < 12; i++) - { - int offset = 5 * i; - e += rotate(a, 5) + f3(b, c, d) + words[offset] + 0x8f1bbcdc; - b = rotate(b, 30); - d += rotate(e, 5) + f3(a, b, c) + words[offset + 1] + 0x8f1bbcdc; - a = rotate(a, 30); - c += rotate(d, 5) + f3(e, a, b) + words[offset + 2] + 0x8f1bbcdc; - e = rotate(e, 30); - b += rotate(c, 5) + f3(d, e, a) + words[offset + 3] + 0x8f1bbcdc; - d = rotate(d, 30); - a += rotate(b, 5) + f3(c, d, e) + words[offset + 4] + 0x8f1bbcdc; - c = rotate(c, 30); - } - - // fourth round - for (int i = 12; i < 16; i++) - { - int offset = 5 * i; - e += rotate(a, 5) + f2(b, c, d) + words[offset] + 0xca62c1d6; - b = rotate(b, 30); - d += rotate(e, 5) + f2(a, b, c) + words[offset + 1] + 0xca62c1d6; - a = rotate(a, 30); - c += rotate(d, 5) + f2(e, a, b) + words[offset + 2] + 0xca62c1d6; - e = rotate(e, 30); - b += rotate(c, 5) + f2(d, e, a) + words[offset + 3] + 0xca62c1d6; - d = rotate(d, 30); - a += rotate(b, 5) + f2(c, d, e) + words[offset + 4] + 0xca62c1d6; - c = rotate(c, 30); - } - - // update hash - m_Hash[0] += a; - m_Hash[1] += b; - m_Hash[2] += c; - m_Hash[3] += d; - m_Hash[4] += e; -} - -/// add arbitrary number of bytes -SHA1Stream& -SHA1Stream::Append(const void* data, size_t byteCount) -{ - const uint8_t* current = (const uint8_t*)data; - - if (m_BufferSize > 0) - { - while (byteCount > 0 && m_BufferSize < BlockSize) - { - m_Buffer[m_BufferSize++] = *current++; - byteCount--; - } - } - - // full buffer - if (m_BufferSize == BlockSize) - { - ProcessBlock((void*)m_Buffer); - m_NumBytes += BlockSize; - m_BufferSize = 0; - } - - // no more data ? - if (byteCount == 0) - return *this; - - // process full blocks - while (byteCount >= BlockSize) - { - ProcessBlock(current); - current += BlockSize; - m_NumBytes += BlockSize; - byteCount -= BlockSize; - } - - // keep remaining bytes in buffer - while (byteCount > 0) - { - m_Buffer[m_BufferSize++] = *current++; - byteCount--; - } - - return *this; -} - -/// process final block, less than 64 bytes -void -SHA1Stream::ProcessBuffer() -{ - // the input bytes are considered as bits strings, where the first bit is the most significant bit of the byte - - // - append "1" bit to message - // - append "0" bits until message length in bit mod 512 is 448 - // - append length as 64 bit integer - - // number of bits - size_t paddedLength = m_BufferSize * 8; - - // plus one bit set to 1 (always appended) - paddedLength++; - - // number of bits must be (numBits % 512) = 448 - size_t lower11Bits = paddedLength & 511; - if (lower11Bits <= 448) - paddedLength += 448 - lower11Bits; - else - paddedLength += 512 + 448 - lower11Bits; - // convert from bits to bytes - paddedLength /= 8; - - // only needed if additional data flows over into a second block - unsigned char extra[BlockSize]; - - // append a "1" bit, 128 => binary 10000000 - if (m_BufferSize < BlockSize) - m_Buffer[m_BufferSize] = 128; - else - extra[0] = 128; - - size_t i; - for (i = m_BufferSize + 1; i < BlockSize; i++) - m_Buffer[i] = 0; - for (; i < paddedLength; i++) - extra[i - BlockSize] = 0; - - // add message length in bits as 64 bit number - uint64_t msgBits = 8 * (m_NumBytes + m_BufferSize); - // find right position - unsigned char* addLength; - if (paddedLength < BlockSize) - addLength = m_Buffer + paddedLength; - else - addLength = extra + paddedLength - BlockSize; - - // must be big endian - *addLength++ = (unsigned char)((msgBits >> 56) & 0xFF); - *addLength++ = (unsigned char)((msgBits >> 48) & 0xFF); - *addLength++ = (unsigned char)((msgBits >> 40) & 0xFF); - *addLength++ = (unsigned char)((msgBits >> 32) & 0xFF); - *addLength++ = (unsigned char)((msgBits >> 24) & 0xFF); - *addLength++ = (unsigned char)((msgBits >> 16) & 0xFF); - *addLength++ = (unsigned char)((msgBits >> 8) & 0xFF); - *addLength = (unsigned char)(msgBits & 0xFF); - - // process blocks - ProcessBlock(m_Buffer); - // flowed over into a second block ? - if (paddedLength > BlockSize) - ProcessBlock(extra); -} - -/// return latest hash as bytes -SHA1 -SHA1Stream::GetHash() -{ - SHA1 sha1; - // save old hash if buffer is partially filled - uint32_t oldHash[HashValues]; - for (int i = 0; i < HashValues; i++) - oldHash[i] = m_Hash[i]; - - // process remaining bytes - ProcessBuffer(); - - unsigned char* current = sha1.Hash; - for (int i = 0; i < HashValues; i++) - { - *current++ = (m_Hash[i] >> 24) & 0xFF; - *current++ = (m_Hash[i] >> 16) & 0xFF; - *current++ = (m_Hash[i] >> 8) & 0xFF; - *current++ = m_Hash[i] & 0xFF; - - // restore old hash - m_Hash[i] = oldHash[i]; - } - - return sha1; -} - -/// compute SHA1 of a memory block -SHA1 -SHA1Stream::Compute(const void* data, size_t byteCount) -{ - Reset(); - Append(data, byteCount); - return GetHash(); -} - -SHA1 -SHA1::HashMemory(const void* data, size_t byteCount) -{ - return SHA1Stream().Append(data, byteCount).GetHash(); -} - -SHA1 -SHA1::FromHexString(const char* string) -{ - SHA1 sha1; - - ParseHexBytes(string, 40, sha1.Hash); - - return sha1; -} - -const char* -SHA1::ToHexString(char* outString /* 40 characters + NUL terminator */) const -{ - ToHexBytes(Hash, sizeof(SHA1), outString); - outString[2 * sizeof(SHA1)] = '\0'; - - return outString; -} - -StringBuilderBase& -SHA1::ToHexString(StringBuilderBase& outBuilder) const -{ - char str[41]; - ToHexString(str); - - outBuilder.AppendRange(str, &str[40]); - - return outBuilder; -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -void -sha1_forcelink() -{ -} - -// doctest::String -// toString(const SHA1& value) -// { -// char sha1text[2 * sizeof(SHA1) + 1]; -// value.ToHexString(sha1text); - -// return sha1text; -// } - -TEST_CASE("SHA1") -{ - uint8_t sha1_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, - 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09}; - SHA1 sha1z; - memcpy(sha1z.Hash, sha1_empty, sizeof sha1z.Hash); - - SUBCASE("Empty string") - { - SHA1 sha1 = SHA1::HashMemory(nullptr, 0); - - CHECK(sha1 == sha1z); - } - - SUBCASE("Empty stream") - { - SHA1Stream sha1s; - sha1s.Append(nullptr, 0); - sha1s.Append(nullptr, 0); - sha1s.Append(nullptr, 0); - CHECK(sha1s.GetHash() == sha1z); - } - - SUBCASE("SHA1 from string") - { - const SHA1 sha1empty = SHA1::FromHexString("da39a3ee5e6b4b0d3255bfef95601890afd80709"); - - CHECK(sha1z == sha1empty); - } - - SUBCASE("SHA1 to string") - { - char sha1str[41]; - sha1z.ToHexString(sha1str); - - CHECK(StringEquals(sha1str, "da39a3ee5e6b4b0d3255bfef95601890afd80709")); - } - - SUBCASE("Hash ABC") - { - const SHA1 sha1abc = SHA1::FromHexString("3c01bdbb26f358bab27f267924aa2c9a03fcfdb8"); - - SHA1Stream sha1s; - - sha1s.Append("A", 1); - sha1s.Append("B", 1); - sha1s.Append("C", 1); - CHECK(sha1s.GetHash() == sha1abc); - - sha1s.Reset(); - sha1s.Append("AB", 2); - sha1s.Append("C", 1); - CHECK(sha1s.GetHash() == sha1abc); - - sha1s.Reset(); - sha1s.Append("ABC", 3); - CHECK(sha1s.GetHash() == sha1abc); - - sha1s.Reset(); - sha1s.Append("A", 1); - sha1s.Append("BC", 2); - CHECK(sha1s.GetHash() == sha1abc); - } -} - -#endif - -} // namespace zen diff --git a/zencore/sharedbuffer.cpp b/zencore/sharedbuffer.cpp deleted file mode 100644 index 200e06972..000000000 --- a/zencore/sharedbuffer.cpp +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/sharedbuffer.h> - -#include <zencore/testing.h> - -#include <memory.h> - -#include <gsl/gsl-lite.hpp> - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -UniqueBuffer -UniqueBuffer::Alloc(uint64_t Size) -{ - void* Buffer = Memory::Alloc(Size, 16); - IoBufferCore* Owner = new IoBufferCore(Buffer, Size); - Owner->SetIsOwnedByThis(true); - Owner->SetIsImmutable(false); - - return UniqueBuffer(Owner); -} - -UniqueBuffer -UniqueBuffer::MakeMutableView(void* DataPtr, uint64_t Size) -{ - IoBufferCore* Owner = new IoBufferCore(DataPtr, Size); - Owner->SetIsImmutable(false); - return UniqueBuffer(Owner); -} - -UniqueBuffer::UniqueBuffer(IoBufferCore* Owner) : m_Buffer(Owner) -{ -} - -SharedBuffer -UniqueBuffer::MoveToShared() -{ - return SharedBuffer(std::move(m_Buffer)); -} - -void -UniqueBuffer::Reset() -{ - m_Buffer = nullptr; -} - -////////////////////////////////////////////////////////////////////////// - -SharedBuffer::SharedBuffer(UniqueBuffer&& InBuffer) : m_Buffer(std::move(InBuffer.m_Buffer)) -{ -} - -SharedBuffer -SharedBuffer::MakeOwned() const& -{ - if (IsOwned() || !m_Buffer) - { - return *this; - } - else - { - return Clone(GetView()); - } -} - -SharedBuffer -SharedBuffer::MakeOwned() && -{ - if (IsOwned()) - { - return std::move(*this); - } - else - { - return Clone(GetView()); - } -} - -SharedBuffer -SharedBuffer::MakeView(MemoryView View, SharedBuffer OuterBuffer) -{ - if (OuterBuffer) - { - ZEN_ASSERT(OuterBuffer.GetView().Contains(View)); - } - - if (View == OuterBuffer.GetView()) - { - // Reference to the full buffer contents, so just return the "outer" - return OuterBuffer; - } - - IoBufferCore* NewCore = new IoBufferCore(OuterBuffer.m_Buffer, View.GetData(), View.GetSize()); - NewCore->SetIsImmutable(true); - return SharedBuffer(NewCore); -} - -SharedBuffer -SharedBuffer::MakeView(const void* Data, uint64_t Size) -{ - return SharedBuffer(new IoBufferCore(const_cast<void*>(Data), Size)); -} - -SharedBuffer -SharedBuffer::Clone() -{ - const uint64_t Size = GetSize(); - void* Buffer = Memory::Alloc(Size, 16); - auto NewOwner = new IoBufferCore(Buffer, Size); - NewOwner->SetIsOwnedByThis(true); - memcpy(Buffer, m_Buffer->DataPointer(), Size); - - return SharedBuffer(NewOwner); -} - -SharedBuffer -SharedBuffer::Clone(MemoryView View) -{ - const uint64_t Size = View.GetSize(); - void* Buffer = Memory::Alloc(Size, 16); - auto NewOwner = new IoBufferCore(Buffer, Size); - NewOwner->SetIsOwnedByThis(true); - memcpy(Buffer, View.GetData(), Size); - - return SharedBuffer(NewOwner); -} - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_WITH_TESTS - -void -sharedbuffer_forcelink() -{ -} - -TEST_CASE("SharedBuffer") -{ -} - -#endif - -} // namespace zen diff --git a/zencore/stats.cpp b/zencore/stats.cpp deleted file mode 100644 index 372bc42f8..000000000 --- a/zencore/stats.cpp +++ /dev/null @@ -1,715 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/stats.h" - -#include <zencore/compactbinarybuilder.h> -#include "zencore/intmath.h" -#include "zencore/thread.h" -#include "zencore/timer.h" - -#include <cmath> -#include <gsl/gsl-lite.hpp> - -#if ZEN_WITH_TESTS -# include <zencore/testing.h> -#endif - -// -// Derived from https://github.com/dln/medida/blob/master/src/medida/stats/ewma.cc -// - -namespace zen::metrics { - -static constinit int kTickIntervalInSeconds = 5; -static constinit double kSecondsPerMinute = 60.0; -static constinit int kOneMinute = 1; -static constinit int kFiveMinutes = 5; -static constinit int kFifteenMinutes = 15; - -static const double kM1_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kOneMinute); -static const double kM5_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kFiveMinutes); -static const double kM15_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kFifteenMinutes); - -static const uint64_t CountPerTick = GetHifreqTimerFrequencySafe() * kTickIntervalInSeconds; -static const uint64_t CountPerSecond = GetHifreqTimerFrequencySafe(); - -////////////////////////////////////////////////////////////////////////// - -void -RawEWMA::Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate) -{ - const double InstantRate = double(Count) / Interval; - - if (IsInitialUpdate) - { - m_Rate.store(InstantRate, std::memory_order_release); - } - else - { - double Delta = Alpha * (InstantRate - m_Rate); - -#if defined(__cpp_lib_atomic_float) - m_Rate.fetch_add(Delta); -#else - double Value = m_Rate.load(std::memory_order_acquire); - double Next; - do - { - Next = Value + Delta; - } while (!m_Rate.compare_exchange_weak(Value, Next, std::memory_order_relaxed)); -#endif - } -} - -double -RawEWMA::Rate() const -{ - return m_Rate.load(std::memory_order_relaxed) * CountPerSecond; -} - -////////////////////////////////////////////////////////////////////////// - -Meter::Meter() : m_StartTick{GetHifreqTimerValue()}, m_LastTick(m_StartTick.load()) -{ -} - -Meter::~Meter() -{ -} - -void -Meter::TickIfNecessary() -{ - uint64_t OldTick = m_LastTick.load(); - const uint64_t NewTick = GetHifreqTimerValue(); - const uint64_t Age = NewTick - OldTick; - - if (Age > CountPerTick) - { - // Ensure only one thread at a time updates the time. This - // works because our tick interval should be sufficiently - // long to ensure two threads don't end up inside this block - - if (m_LastTick.compare_exchange_strong(OldTick, NewTick)) - { - m_Remainder.fetch_add(Age); - - do - { - int64_t Remain = m_Remainder.load(std::memory_order_relaxed); - - if (Remain < 0) - { - return; - } - - if (m_Remainder.compare_exchange_strong(Remain, Remain - CountPerTick)) - { - Tick(); - } - } while (true); - } - } -} - -void -Meter::Tick() -{ - const uint64_t PendingCount = m_PendingCount.exchange(0); - const bool IsFirstTick = m_IsFirstTick; - - if (IsFirstTick) - { - m_IsFirstTick = false; - } - - m_RateM1.Tick(kM1_ALPHA, CountPerTick, PendingCount, IsFirstTick); - m_RateM5.Tick(kM5_ALPHA, CountPerTick, PendingCount, IsFirstTick); - m_RateM15.Tick(kM15_ALPHA, CountPerTick, PendingCount, IsFirstTick); -} - -double -Meter::Rate1() -{ - TickIfNecessary(); - - return m_RateM1.Rate(); -} - -double -Meter::Rate5() -{ - TickIfNecessary(); - - return m_RateM5.Rate(); -} - -double -Meter::Rate15() -{ - TickIfNecessary(); - - return m_RateM15.Rate(); -} - -double -Meter::MeanRate() const -{ - const uint64_t Count = m_TotalCount.load(std::memory_order_relaxed); - - if (Count == 0) - { - return 0.0; - } - - const uint64_t Elapsed = GetHifreqTimerValue() - m_StartTick; - - return (double(Count) * GetHifreqTimerFrequency()) / Elapsed; -} - -void -Meter::Mark(uint64_t Count) -{ - TickIfNecessary(); - - m_TotalCount.fetch_add(Count); - m_PendingCount.fetch_add(Count); -} - -////////////////////////////////////////////////////////////////////////// - -// TODO: should consider a cheaper RNG here, this will run for every thread -// that gets created - -thread_local std::mt19937_64 ThreadLocalRng; - -UniformSample::UniformSample(uint32_t ReservoirSize) : m_Values(ReservoirSize) -{ -} - -UniformSample::~UniformSample() -{ -} - -void -UniformSample::Clear() -{ - for (auto& Value : m_Values) - { - Value.store(0); - } - m_SampleCounter = 0; -} - -uint32_t -UniformSample::Size() const -{ - return gsl::narrow_cast<uint32_t>(Min(m_SampleCounter.load(), m_Values.size())); -} - -void -UniformSample::Update(int64_t Value) -{ - const uint64_t Count = m_SampleCounter++; - const uint64_t Size = m_Values.size(); - - if (Count < Size) - { - m_Values[Count] = Value; - } - else - { - // Randomly choose an old entry to potentially replace (the probability - // of replacing an entry diminishes with time) - - std::uniform_int_distribution<uint64_t> UniformDist(0, Count); - uint64_t SampleIndex = UniformDist(ThreadLocalRng); - - if (SampleIndex < Size) - { - m_Values[SampleIndex].store(Value, std::memory_order_release); - } - } -} - -SampleSnapshot -UniformSample::Snapshot() const -{ - uint64_t ValuesSize = Size(); - std::vector<double> Values(ValuesSize); - - for (int i = 0, n = int(ValuesSize); i < n; ++i) - { - Values[i] = double(m_Values[i]); - } - - return SampleSnapshot(std::move(Values)); -} - -////////////////////////////////////////////////////////////////////////// - -Histogram::Histogram(int32_t SampleCount) : m_Sample(SampleCount) -{ -} - -Histogram::~Histogram() -{ -} - -void -Histogram::Clear() -{ - m_Min = m_Max = m_Sum = m_Count = 0; - m_Sample.Clear(); -} - -void -Histogram::Update(int64_t Value) -{ - m_Sample.Update(Value); - - if (m_Count == 0) - { - m_Min = m_Max = Value; - } - else - { - int64_t CurrentMax = m_Max.load(std::memory_order_relaxed); - - while ((CurrentMax < Value) && !m_Max.compare_exchange_weak(CurrentMax, Value)) - { - } - - int64_t CurrentMin = m_Min.load(std::memory_order_relaxed); - - while ((CurrentMin > Value) && !m_Min.compare_exchange_weak(CurrentMin, Value)) - { - } - } - - m_Sum += Value; - ++m_Count; -} - -int64_t -Histogram::Max() const -{ - return m_Max.load(std::memory_order_relaxed); -} - -int64_t -Histogram::Min() const -{ - return m_Min.load(std::memory_order_relaxed); -} - -double -Histogram::Mean() const -{ - if (m_Count) - { - return double(m_Sum.load(std::memory_order_relaxed)) / m_Count; - } - else - { - return 0.0; - } -} - -uint64_t -Histogram::Count() const -{ - return m_Count.load(std::memory_order_relaxed); -} - -////////////////////////////////////////////////////////////////////////// - -SampleSnapshot::SampleSnapshot(std::vector<double>&& Values) : m_Values(std::move(Values)) -{ - std::sort(begin(m_Values), end(m_Values)); -} - -SampleSnapshot::~SampleSnapshot() -{ -} - -double -SampleSnapshot::GetQuantileValue(double Quantile) -{ - ZEN_ASSERT((Quantile >= 0.0) && (Quantile <= 1.0)); - - if (m_Values.empty()) - { - return 0.0; - } - - const double Pos = Quantile * (m_Values.size() + 1); - - if (Pos < 1) - { - return m_Values.front(); - } - - if (Pos >= m_Values.size()) - { - return m_Values.back(); - } - - const int32_t Index = (int32_t)Pos; - const double Lower = m_Values[Index - 1]; - const double Upper = m_Values[Index]; - - // Lerp - return Lower + (Pos - std::floor(Pos)) * (Upper - Lower); -} - -const std::vector<double>& -SampleSnapshot::GetValues() const -{ - return m_Values; -} - -////////////////////////////////////////////////////////////////////////// - -OperationTiming::OperationTiming(int32_t SampleCount) : m_Histogram{SampleCount} -{ -} - -OperationTiming::~OperationTiming() -{ -} - -void -OperationTiming::Update(int64_t Duration) -{ - m_Meter.Mark(1); - m_Histogram.Update(Duration); -} - -int64_t -OperationTiming::Max() const -{ - return m_Histogram.Max(); -} - -int64_t -OperationTiming::Min() const -{ - return m_Histogram.Min(); -} - -double -OperationTiming::Mean() const -{ - return m_Histogram.Mean(); -} - -uint64_t -OperationTiming::Count() const -{ - return m_Meter.Count(); -} - -OperationTiming::Scope::Scope(OperationTiming& Outer) : m_Outer(Outer), m_StartTick(GetHifreqTimerValue()) -{ -} - -OperationTiming::Scope::~Scope() -{ - Stop(); -} - -void -OperationTiming::Scope::Stop() -{ - if (m_StartTick != 0) - { - m_Outer.Update(GetHifreqTimerValue() - m_StartTick); - m_StartTick = 0; - } -} - -void -OperationTiming::Scope::Cancel() -{ - m_StartTick = 0; -} - -////////////////////////////////////////////////////////////////////////// - -RequestStats::RequestStats(int32_t SampleCount) : m_RequestTimeHistogram{SampleCount}, m_BytesHistogram{SampleCount} -{ -} - -RequestStats::~RequestStats() -{ -} - -void -RequestStats::Update(int64_t Duration, int64_t Bytes) -{ - m_RequestMeter.Mark(1); - m_RequestTimeHistogram.Update(Duration); - - m_BytesMeter.Mark(Bytes); - m_BytesHistogram.Update(Bytes); -} - -uint64_t -RequestStats::Count() const -{ - return m_RequestMeter.Count(); -} - -////////////////////////////////////////////////////////////////////////// - -void -EmitSnapshot(Meter& Stat, CbObjectWriter& Cbo) -{ - Cbo << "count" << Stat.Count(); - Cbo << "rate_mean" << Stat.MeanRate(); - Cbo << "rate_1" << Stat.Rate1() << "rate_5" << Stat.Rate5() << "rate_15" << Stat.Rate15(); -} - -void -RequestStats::EmitSnapshot(std::string_view Tag, CbObjectWriter& Cbo) -{ - Cbo.BeginObject(Tag); - - Cbo.BeginObject("requests"); - metrics::EmitSnapshot(m_RequestMeter, Cbo); - metrics::EmitSnapshot(m_RequestTimeHistogram, Cbo, GetHifreqTimerToSeconds()); - Cbo.EndObject(); - - Cbo.BeginObject("bytes"); - metrics::EmitSnapshot(m_BytesMeter, Cbo); - metrics::EmitSnapshot(m_BytesHistogram, Cbo, 1.0); - Cbo.EndObject(); - - Cbo.EndObject(); -} - -void -EmitSnapshot(std::string_view Tag, OperationTiming& Stat, CbObjectWriter& Cbo) -{ - Cbo.BeginObject(Tag); - - SampleSnapshot Snap = Stat.Snapshot(); - - Cbo << "count" << Stat.Count(); - Cbo << "rate_mean" << Stat.MeanRate(); - Cbo << "rate_1" << Stat.Rate1() << "rate_5" << Stat.Rate5() << "rate_15" << Stat.Rate15(); - - const double ToSeconds = GetHifreqTimerToSeconds(); - - Cbo << "t_avg" << Stat.Mean() * ToSeconds; - Cbo << "t_min" << Stat.Min() * ToSeconds << "t_max" << Stat.Max() * ToSeconds; - Cbo << "t_p75" << Snap.Get75Percentile() * ToSeconds << "t_p95" << Snap.Get95Percentile() * ToSeconds << "t_p99" - << Snap.Get99Percentile() * ToSeconds << "t_p999" << Snap.Get999Percentile() * ToSeconds; - - Cbo.EndObject(); -} - -void -EmitSnapshot(std::string_view Tag, const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor) -{ - Cbo.BeginObject(Tag); - EmitSnapshot(Stat, Cbo, ConversionFactor); - Cbo.EndObject(); -} - -void -EmitSnapshot(const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor) -{ - SampleSnapshot Snap = Stat.Snapshot(); - - Cbo << "count" << Stat.Count() * ConversionFactor << "avg" << Stat.Mean() * ConversionFactor; - Cbo << "min" << Stat.Min() * ConversionFactor << "max" << Stat.Max() * ConversionFactor; - Cbo << "p75" << Snap.Get75Percentile() * ConversionFactor << "p95" << Snap.Get95Percentile() * ConversionFactor << "p99" - << Snap.Get99Percentile() * ConversionFactor << "p999" << Snap.Get999Percentile() * ConversionFactor; -} - -void -EmitSnapshot(std::string_view Tag, Meter& Stat, CbObjectWriter& Cbo) -{ - Cbo.BeginObject(Tag); - - Cbo << "count" << Stat.Count() << "rate_mean" << Stat.MeanRate(); - Cbo << "rate_1" << Stat.Rate1() << "rate_5" << Stat.Rate5() << "rate_15" << Stat.Rate15(); - - Cbo.EndObject(); -} - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_WITH_TESTS - -TEST_CASE("Core.Stats.Histogram") -{ - Histogram Histo{258}; - - SampleSnapshot Snap1 = Histo.Snapshot(); - CHECK_EQ(Snap1.Size(), 0); - CHECK_EQ(Snap1.GetMedian(), 0); - - Histo.Update(1); - CHECK_EQ(Histo.Min(), 1); - CHECK_EQ(Histo.Max(), 1); - - SampleSnapshot Snap2 = Histo.Snapshot(); - CHECK_EQ(Snap2.Size(), 1); - - Histo.Update(2); - CHECK_EQ(Histo.Min(), 1); - CHECK_EQ(Histo.Max(), 2); - - SampleSnapshot Snap3 = Histo.Snapshot(); - CHECK_EQ(Snap3.Size(), 2); - - Histo.Update(-2); - CHECK_EQ(Histo.Min(), -2); - CHECK_EQ(Histo.Max(), 2); - CHECK_EQ(Histo.Mean(), 1 / 3.0); - - SampleSnapshot Snap4 = Histo.Snapshot(); - CHECK_EQ(Snap4.Size(), 3); - CHECK_EQ(Snap4.GetMedian(), 1); - CHECK_EQ(Snap4.Get999Percentile(), 2); - CHECK_EQ(Snap4.GetQuantileValue(0), -2); -} - -TEST_CASE("Core.Stats.UniformSample") -{ - UniformSample Sample1{100}; - - for (int i = 0; i < 100; ++i) - { - for (int j = 1; j <= 100; ++j) - { - Sample1.Update(j); - } - } - - int64_t Sum = 0; - int64_t Count = 0; - - Sample1.IterateValues([&](int64_t Value) { - ++Count; - Sum += Value; - }); - - double Average = double(Sum) / Count; - - CHECK(fabs(Average - 50) < 10); // What's the right test here? The result could vary massively and still be technically correct -} - -TEST_CASE("Core.Stats.EWMA") -{ - SUBCASE("Simple_1") - { - RawEWMA Ewma1; - Ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true); - - CHECK(fabs(Ewma1.Rate() - 5) < 0.1); - - for (int i = 0; i < 60; ++i) - { - Ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false); - } - - CHECK(fabs(Ewma1.Rate() - 10) < 0.1); - - for (int i = 0; i < 60; ++i) - { - Ewma1.Tick(kM1_ALPHA, CountPerSecond, 20, false); - } - - CHECK(fabs(Ewma1.Rate() - 20) < 0.1); - } - - SUBCASE("Simple_10") - { - RawEWMA Ewma1; - RawEWMA Ewma5; - RawEWMA Ewma15; - Ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true); - Ewma5.Tick(kM5_ALPHA, CountPerSecond, 5, true); - Ewma15.Tick(kM15_ALPHA, CountPerSecond, 5, true); - - CHECK(fabs(Ewma1.Rate() - 5) < 0.1); - CHECK(fabs(Ewma5.Rate() - 5) < 0.1); - CHECK(fabs(Ewma15.Rate() - 5) < 0.1); - - auto Tick1 = [&Ewma1](auto Value) { Ewma1.Tick(kM1_ALPHA, CountPerSecond, Value, false); }; - auto Tick5 = [&Ewma5](auto Value) { Ewma5.Tick(kM5_ALPHA, CountPerSecond, Value, false); }; - auto Tick15 = [&Ewma15](auto Value) { Ewma15.Tick(kM15_ALPHA, CountPerSecond, Value, false); }; - - for (int i = 0; i < 60; ++i) - { - Tick1(10); - Tick5(10); - Tick15(10); - } - - CHECK(fabs(Ewma1.Rate() - 10) < 0.1); - - for (int i = 0; i < 5 * 60; ++i) - { - Tick1(20); - Tick5(20); - Tick15(20); - } - - CHECK(fabs(Ewma1.Rate() - 20) < 0.1); - CHECK(fabs(Ewma5.Rate() - 20) < 0.1); - - for (int i = 0; i < 16 * 60; ++i) - { - Tick1(100); - Tick5(100); - Tick15(100); - } - - CHECK(fabs(Ewma1.Rate() - 100) < 0.1); - CHECK(fabs(Ewma5.Rate() - 100) < 0.1); - CHECK(fabs(Ewma15.Rate() - 100) < 0.5); - } -} - -# if 0 // This is not really a unit test, but mildly useful to exercise some code -TEST_CASE("Meter") -{ - Meter Meter1; - Meter1.Mark(1); - Sleep(1000); - Meter1.Mark(1); - Sleep(1000); - Meter1.Mark(1); - Sleep(1000); - Meter1.Mark(1); - Sleep(1000); - Meter1.Mark(1); - Sleep(1000); - Meter1.Mark(1); - Sleep(1000); - Meter1.Mark(1); - Sleep(1000); - Meter1.Mark(1); - Sleep(1000); - Meter1.Mark(1); - Sleep(1000); - [[maybe_unused]] double Rate = Meter1.MeanRate(); -} -# endif -} - -namespace zen { - -void -stats_forcelink() -{ -} - -#endif - -} // namespace zen::metrics diff --git a/zencore/stream.cpp b/zencore/stream.cpp deleted file mode 100644 index 3402e51be..000000000 --- a/zencore/stream.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <stdarg.h> -#include <zencore/memory.h> -#include <zencore/stream.h> -#include <zencore/testing.h> - -#include <algorithm> -#include <stdexcept> - -namespace zen { - -void -BinaryWriter::Write(std::initializer_list<const MemoryView> Buffers) -{ - size_t TotalByteCount = 0; - for (const MemoryView& View : Buffers) - { - TotalByteCount += View.GetSize(); - } - const size_t NeedEnd = m_Offset + TotalByteCount; - if (NeedEnd > m_Buffer.size()) - { - m_Buffer.resize(NeedEnd); - } - for (const MemoryView& View : Buffers) - { - memcpy(m_Buffer.data() + m_Offset, View.GetData(), View.GetSize()); - m_Offset += View.GetSize(); - } -} - -void -BinaryWriter::Write(const void* data, size_t ByteCount, uint64_t Offset) -{ - const size_t NeedEnd = Offset + ByteCount; - - if (NeedEnd > m_Buffer.size()) - { - m_Buffer.resize(NeedEnd); - } - - memcpy(m_Buffer.data() + Offset, data, ByteCount); -} - -void -BinaryWriter::Reset() -{ - m_Buffer.clear(); - m_Offset = 0; -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -TEST_CASE("binary.writer.span") -{ - BinaryWriter Writer; - const MemoryView View1("apa", 3); - const MemoryView View2(" ", 1); - const MemoryView View3("banan", 5); - Writer.Write({View1, View2, View3}); - MemoryView Result = Writer.GetView(); - CHECK(Result.GetSize() == 9); - CHECK(memcmp(Result.GetData(), "apa banan", 9) == 0); -} - -void -stream_forcelink() -{ -} - -#endif - -} // namespace zen diff --git a/zencore/string.cpp b/zencore/string.cpp deleted file mode 100644 index ad6ee78fc..000000000 --- a/zencore/string.cpp +++ /dev/null @@ -1,1004 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/memory.h> -#include <zencore/string.h> -#include <zencore/testing.h> - -#include <inttypes.h> -#include <math.h> -#include <stdio.h> -#include <exception> -#include <ostream> -#include <stdexcept> - -#include <utf8.h> - -template<typename u16bit_iterator> -void -utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuilderBase& OutString) -{ - while (StartIt != EndIt) - { - uint32_t cp = utf8::internal::mask16(*StartIt++); - // Take care of surrogate pairs first - if (utf8::internal::is_lead_surrogate(cp)) - { - uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++); - cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET; - } - OutString.AppendCodepoint(cp); - } -} - -template<typename u32bit_iterator> -void -utf32to8_impl(u32bit_iterator StartIt, u32bit_iterator EndIt, ::zen::StringBuilderBase& OutString) -{ - for (; StartIt != EndIt; ++StartIt) - { - wchar_t cp = *StartIt; - OutString.AppendCodepoint(cp); - } -} - -////////////////////////////////////////////////////////////////////////// - -namespace zen { - -bool -ToString(std::span<char> Buffer, uint64_t Num) -{ - snprintf(Buffer.data(), Buffer.size(), "%" PRIu64, Num); - - return true; -} -bool -ToString(std::span<char> Buffer, int64_t Num) -{ - snprintf(Buffer.data(), Buffer.size(), "%" PRId64, Num); - - return true; -} - -////////////////////////////////////////////////////////////////////////// - -const char* -FilepathFindExtension(const std::string_view& Path, const char* ExtensionToMatch) -{ - const size_t PathLen = Path.size(); - - if (ExtensionToMatch) - { - size_t ExtLen = strlen(ExtensionToMatch); - - if (ExtLen > PathLen) - return nullptr; - - const char* PathExtension = Path.data() + PathLen - ExtLen; - - if (StringEquals(PathExtension, ExtensionToMatch)) - return PathExtension; - - return nullptr; - } - - if (PathLen == 0) - return nullptr; - - // Look for extension introducer ('.') - - for (int64_t i = PathLen - 1; i >= 0; --i) - { - if (Path[i] == '.') - return Path.data() + i; - } - - return nullptr; -} - -////////////////////////////////////////////////////////////////////////// - -void -Utf8ToWide(const char8_t* Str8, WideStringBuilderBase& OutString) -{ - Utf8ToWide(std::u8string_view(Str8), OutString); -} - -void -Utf8ToWide(const std::string_view& Str8, WideStringBuilderBase& OutString) -{ - Utf8ToWide(std::u8string_view{reinterpret_cast<const char8_t*>(Str8.data()), Str8.size()}, OutString); -} - -std::wstring -Utf8ToWide(const std::string_view& Wstr) -{ - ExtendableWideStringBuilder<128> String; - Utf8ToWide(Wstr, String); - - return String.c_str(); -} - -void -Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString) -{ - const char* str = (const char*)Str8.data(); - const size_t strLen = Str8.size(); - - const char* endStr = str + strLen; - size_t ByteCount = 0; - size_t CurrentOutChar = 0; - - for (; str != endStr; ++str) - { - unsigned char Data = static_cast<unsigned char>(*str); - - if (!(Data & 0x80)) - { - // ASCII - OutString.Append(wchar_t(Data)); - continue; - } - else if (!ByteCount) - { - // Start of multi-byte sequence. Figure out how - // many bytes we're going to consume - - size_t Count = 0; - - for (size_t Temp = Data; Temp & 0x80; Temp <<= 1) - ++Count; - - ByteCount = Count - 1; - CurrentOutChar = Data & (0xff >> (Count + 1)); - } - else - { - --ByteCount; - - if ((Data & 0xc0) != 0x80) - { - break; - } - - CurrentOutChar = (CurrentOutChar << 6) | (Data & 0x3f); - - if (!ByteCount) - { - OutString.Append(wchar_t(CurrentOutChar)); - CurrentOutChar = 0; - } - } - } -} - -void -WideToUtf8(const wchar_t* Wstr, StringBuilderBase& OutString) -{ - WideToUtf8(std::wstring_view{Wstr}, OutString); -} - -void -WideToUtf8(const std::wstring_view& Wstr, StringBuilderBase& OutString) -{ -#if ZEN_SIZEOF_WCHAR_T == 2 - utf16to8_impl(begin(Wstr), end(Wstr), OutString); -#else - utf32to8_impl(begin(Wstr), end(Wstr), OutString); -#endif -} - -std::string -WideToUtf8(const wchar_t* Wstr) -{ - ExtendableStringBuilder<128> String; - WideToUtf8(std::wstring_view{Wstr}, String); - - return String.c_str(); -} - -std::string -WideToUtf8(const std::wstring_view Wstr) -{ - ExtendableStringBuilder<128> String; - WideToUtf8(std::wstring_view{Wstr.data(), Wstr.size()}, String); - - return String.c_str(); -} - -////////////////////////////////////////////////////////////////////////// - -enum NicenumFormat -{ - kNicenum1024 = 0, // Print kilo, mega, tera, peta, exa.. - kNicenumBytes = 1, // Print single bytes ("13B"), kilo, mega, tera... - kNicenumTime = 2, // Print nanosecs, microsecs, millisecs, seconds... - kNicenumRaw = 3, // Print the raw number without any formatting - kNicenumRawTime = 4 // Same as RAW, but print dashes ('-') for zero. -}; - -namespace { - static const char* UnitStrings[3][7] = { - /* kNicenum1024 */ {"", "K", "M", "G", "T", "P", "E"}, - /* kNicenumBytes */ {"B", "K", "M", "G", "T", "P", "E"}, - /* kNicenumTime */ {"ns", "us", "ms", "s", "?", "?", "?"}}; - - static const int UnitsLen[] = { - /* kNicenum1024 */ 6, - /* kNicenumBytes */ 6, - /* kNicenumTime */ 3}; - - static const uint64_t KiloUnit[] = { - /* kNicenum1024 */ 1024, - /* kNicenumBytes */ 1024, - /* kNicenumTime */ 1000}; -} // namespace - -/* - * Convert a number to an appropriately human-readable output. - */ -int -NiceNumGeneral(uint64_t Num, std::span<char> Buffer, NicenumFormat Format) -{ - switch (Format) - { - case kNicenumRaw: - return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64, (uint64_t)Num); - - case kNicenumRawTime: - if (Num > 0) - { - return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64, (uint64_t)Num); - } - else - { - return snprintf(Buffer.data(), Buffer.size(), "%s", "-"); - } - break; - - case kNicenum1024: - case kNicenumBytes: - case kNicenumTime: - default: - break; - } - - // Bring into range and select unit - - int Index = 0; - uint64_t n = Num; - - { - const uint64_t Unit = KiloUnit[Format]; - const int maxIndex = UnitsLen[Format]; - - while (n >= Unit && Index < maxIndex) - { - n /= Unit; - Index++; - } - } - - const char* u = UnitStrings[Format][Index]; - - if ((Index == 0) || ((Num % (uint64_t)powl((int)KiloUnit[Format], Index)) == 0)) - { - /* - * If this is an even multiple of the base, always display - * without any decimal precision. - */ - return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "%s", (uint64_t)n, u); - } - else - { - /* - * We want to choose a precision that reflects the best choice - * for fitting in 5 characters. This can get rather tricky when - * we have numbers that are very close to an order of magnitude. - * For example, when displaying 10239 (which is really 9.999K), - * we want only a single place of precision for 10.0K. We could - * develop some complex heuristics for this, but it's much - * easier just to try each combination in turn. - */ - - int StrLen = 0; - - for (int i = 2; i >= 0; i--) - { - double Value = (double)Num / (uint64_t)powl((int)KiloUnit[Format], Index); - - /* - * Don't print floating point values for time. Note, - * we use floor() instead of round() here, since - * round can result in undesirable results. For - * example, if "num" is in the range of - * 999500-999999, it will print out "1000us". This - * doesn't happen if we use floor(). - */ - if (Format == kNicenumTime) - { - StrLen = snprintf(Buffer.data(), Buffer.size(), "%d%s", (unsigned int)floor(Value), u); - - if (StrLen <= 5) - break; - } - else - { - StrLen = snprintf(Buffer.data(), Buffer.size(), "%.*f%s", i, Value, u); - - if (StrLen <= 5) - break; - } - } - - return StrLen; - } -} - -size_t -NiceNumToBuffer(uint64_t Num, std::span<char> Buffer) -{ - return NiceNumGeneral(Num, Buffer, kNicenum1024); -} - -size_t -NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer) -{ - return NiceNumGeneral(Num, Buffer, kNicenumBytes); -} - -size_t -NiceByteRateToBuffer(uint64_t Num, uint64_t ElapsedMs, std::span<char> Buffer) -{ - size_t n = 0; - - if (ElapsedMs) - { - n = NiceNumGeneral(Num * 1000 / ElapsedMs, Buffer, kNicenumBytes); - } - else - { - Buffer[n++] = '0'; - Buffer[n++] = 'B'; - } - - Buffer[n++] = '/'; - Buffer[n++] = 's'; - Buffer[n++] = '\0'; - - return n; -} - -size_t -NiceLatencyNsToBuffer(uint64_t Nanos, std::span<char> Buffer) -{ - return NiceNumGeneral(Nanos, Buffer, kNicenumTime); -} - -size_t -NiceTimeSpanMsToBuffer(uint64_t Millis, std::span<char> Buffer) -{ - if (Millis < 1000) - { - return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "ms", Millis); - } - else if (Millis < 10000) - { - return snprintf(Buffer.data(), Buffer.size(), "%.2fs", Millis / 1000.0); - } - else if (Millis < 60000) - { - return snprintf(Buffer.data(), Buffer.size(), "%.1fs", Millis / 1000.0); - } - else if (Millis < 60 * 60000) - { - return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "m%02" PRIu64 "s", Millis / 60000, (Millis / 1000) % 60); - } - else - { - return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "h%02" PRIu64 "m", Millis / 3600000, (Millis / 60000) % 60); - } -} - -////////////////////////////////////////////////////////////////////////// - -template<typename C> -StringBuilderImpl<C>::~StringBuilderImpl() -{ - if (m_IsDynamic) - { - FreeBuffer(m_Base, m_End - m_Base); - } -} - -template<typename C> -void -StringBuilderImpl<C>::Extend(size_t extraCapacity) -{ - if (!m_IsExtendable) - { - Fail("exceeded capacity"); - } - - const size_t oldCapacity = m_End - m_Base; - const size_t newCapacity = NextPow2(oldCapacity + extraCapacity); - - C* newBase = (C*)AllocBuffer(newCapacity); - - size_t pos = m_CurPos - m_Base; - memcpy(newBase, m_Base, pos * sizeof(C)); - - if (m_IsDynamic) - { - FreeBuffer(m_Base, oldCapacity); - } - - m_Base = newBase; - m_CurPos = newBase + pos; - m_End = newBase + newCapacity; - m_IsDynamic = true; -} - -template<typename C> -void* -StringBuilderImpl<C>::AllocBuffer(size_t byteCount) -{ - return Memory::Alloc(byteCount * sizeof(C)); -} - -template<typename C> -void -StringBuilderImpl<C>::FreeBuffer(void* buffer, size_t byteCount) -{ - ZEN_UNUSED(byteCount); - - Memory::Free(buffer); -} - -template<typename C> -[[noreturn]] void -StringBuilderImpl<C>::Fail(const char* reason) -{ - throw std::runtime_error(reason); -} - -// Instantiate templates once - -template class StringBuilderImpl<char>; -template class StringBuilderImpl<wchar_t>; - -////////////////////////////////////////////////////////////////////////// -// -// Unit tests -// - -#if ZEN_WITH_TESTS - -TEST_CASE("niceNum") -{ - char Buffer[16]; - - SUBCASE("raw") - { - NiceNumGeneral(1, Buffer, kNicenumRaw); - CHECK(StringEquals(Buffer, "1")); - - NiceNumGeneral(10, Buffer, kNicenumRaw); - CHECK(StringEquals(Buffer, "10")); - - NiceNumGeneral(100, Buffer, kNicenumRaw); - CHECK(StringEquals(Buffer, "100")); - - NiceNumGeneral(1000, Buffer, kNicenumRaw); - CHECK(StringEquals(Buffer, "1000")); - - NiceNumGeneral(10000, Buffer, kNicenumRaw); - CHECK(StringEquals(Buffer, "10000")); - - NiceNumGeneral(100000, Buffer, kNicenumRaw); - CHECK(StringEquals(Buffer, "100000")); - } - - SUBCASE("1024") - { - NiceNumGeneral(1, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1")); - - NiceNumGeneral(10, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "10")); - - NiceNumGeneral(100, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "100")); - - NiceNumGeneral(1000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1000")); - - NiceNumGeneral(10000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "9.77K")); - - NiceNumGeneral(100000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "97.7K")); - - NiceNumGeneral(1000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "977K")); - - NiceNumGeneral(10000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "9.54M")); - - NiceNumGeneral(100000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "95.4M")); - - NiceNumGeneral(1000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "954M")); - - NiceNumGeneral(10000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "9.31G")); - - NiceNumGeneral(100000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "93.1G")); - - NiceNumGeneral(1000000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "931G")); - - NiceNumGeneral(10000000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "9.09T")); - - NiceNumGeneral(100000000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "90.9T")); - - NiceNumGeneral(1000000000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "909T")); - - NiceNumGeneral(10000000000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "8.88P")); - - NiceNumGeneral(100000000000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "88.8P")); - - NiceNumGeneral(1000000000000000000, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "888P")); - - NiceNumGeneral(10000000000000000000ull, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "8.67E")); - - // pow2 - - NiceNumGeneral(0, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "0")); - - NiceNumGeneral(1, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1")); - - NiceNumGeneral(1024, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1K")); - - NiceNumGeneral(1024 * 1024, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1M")); - - NiceNumGeneral(1024 * 1024 * 1024, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1G")); - - NiceNumGeneral(1024llu * 1024 * 1024 * 1024, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1T")); - - NiceNumGeneral(1024llu * 1024 * 1024 * 1024 * 1024, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1P")); - - NiceNumGeneral(1024llu * 1024 * 1024 * 1024 * 1024 * 1024, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1E")); - - // pow2-1 - - NiceNumGeneral(1023, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "1023")); - - NiceNumGeneral(2047, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "2.00K")); - - NiceNumGeneral(9 * 1024 - 1, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "9.00K")); - - NiceNumGeneral(10 * 1024 - 1, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "10.0K")); - - NiceNumGeneral(10 * 1024 - 5, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "10.0K")); - - NiceNumGeneral(10 * 1024 - 6, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "9.99K")); - - NiceNumGeneral(10 * 1024 - 10, Buffer, kNicenum1024); - CHECK(StringEquals(Buffer, "9.99K")); - } - - SUBCASE("time") - { - NiceNumGeneral(1, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "1ns")); - - NiceNumGeneral(100, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "100ns")); - - NiceNumGeneral(1000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "1us")); - - NiceNumGeneral(10000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "10us")); - - NiceNumGeneral(100000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "100us")); - - NiceNumGeneral(1000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "1ms")); - - NiceNumGeneral(10000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "10ms")); - - NiceNumGeneral(100000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "100ms")); - - NiceNumGeneral(1000000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "1s")); - - NiceNumGeneral(10000000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "10s")); - - NiceNumGeneral(100000000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "100s")); - - NiceNumGeneral(1000000000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "1000s")); - - NiceNumGeneral(10000000000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "10000s")); - - NiceNumGeneral(100000000000000, Buffer, kNicenumTime); - CHECK(StringEquals(Buffer, "100000s")); - } - - SUBCASE("bytes") - { - NiceNumGeneral(1, Buffer, kNicenumBytes); - CHECK(StringEquals(Buffer, "1B")); - - NiceNumGeneral(10, Buffer, kNicenumBytes); - CHECK(StringEquals(Buffer, "10B")); - - NiceNumGeneral(100, Buffer, kNicenumBytes); - CHECK(StringEquals(Buffer, "100B")); - - NiceNumGeneral(1000, Buffer, kNicenumBytes); - CHECK(StringEquals(Buffer, "1000B")); - - NiceNumGeneral(10000, Buffer, kNicenumBytes); - CHECK(StringEquals(Buffer, "9.77K")); - } - - SUBCASE("byteRate") - { - NiceByteRateToBuffer(1, 1, Buffer); - CHECK(StringEquals(Buffer, "1000B/s")); - - NiceByteRateToBuffer(1000, 1000, Buffer); - CHECK(StringEquals(Buffer, "1000B/s")); - - NiceByteRateToBuffer(1024, 1, Buffer); - CHECK(StringEquals(Buffer, "1000K/s")); - - NiceByteRateToBuffer(1024, 1000, Buffer); - CHECK(StringEquals(Buffer, "1K/s")); - } - - SUBCASE("timespan") - { - NiceTimeSpanMsToBuffer(1, Buffer); - CHECK(StringEquals(Buffer, "1ms")); - - NiceTimeSpanMsToBuffer(900, Buffer); - CHECK(StringEquals(Buffer, "900ms")); - - NiceTimeSpanMsToBuffer(1000, Buffer); - CHECK(StringEquals(Buffer, "1.00s")); - - NiceTimeSpanMsToBuffer(1900, Buffer); - CHECK(StringEquals(Buffer, "1.90s")); - - NiceTimeSpanMsToBuffer(19000, Buffer); - CHECK(StringEquals(Buffer, "19.0s")); - - NiceTimeSpanMsToBuffer(60000, Buffer); - CHECK(StringEquals(Buffer, "1m00s")); - - NiceTimeSpanMsToBuffer(600000, Buffer); - CHECK(StringEquals(Buffer, "10m00s")); - - NiceTimeSpanMsToBuffer(3600000, Buffer); - CHECK(StringEquals(Buffer, "1h00m")); - - NiceTimeSpanMsToBuffer(36000000, Buffer); - CHECK(StringEquals(Buffer, "10h00m")); - - NiceTimeSpanMsToBuffer(360000000, Buffer); - CHECK(StringEquals(Buffer, "100h00m")); - } -} - -void -string_forcelink() -{ -} - -TEST_CASE("StringBuilder") -{ - StringBuilder<64> sb; - - SUBCASE("Empty init") - { - const char* str = sb.c_str(); - - CHECK(StringLength(str) == 0); - } - - SUBCASE("Append single character") - { - sb.Append('a'); - - const char* str = sb.c_str(); - CHECK(StringLength(str) == 1); - CHECK(str[0] == 'a'); - - sb.Append('b'); - str = sb.c_str(); - CHECK(StringLength(str) == 2); - CHECK(str[0] == 'a'); - CHECK(str[1] == 'b'); - } - - SUBCASE("Append string") - { - sb.Append("a"); - - const char* str = sb.c_str(); - CHECK(StringLength(str) == 1); - CHECK(str[0] == 'a'); - - sb.Append("b"); - str = sb.c_str(); - CHECK(StringLength(str) == 2); - CHECK(str[0] == 'a'); - CHECK(str[1] == 'b'); - - sb.Append("cdefghijklmnopqrstuvwxyz"); - CHECK(sb.Size() == 26); - - sb.Append("abcdefghijklmnopqrstuvwxyz"); - CHECK(sb.Size() == 52); - - sb.Append("abcdefghijk"); - CHECK(sb.Size() == 63); - } -} - -TEST_CASE("ExtendableStringBuilder") -{ - ExtendableStringBuilder<16> sb; - - SUBCASE("Empty init") - { - const char* str = sb.c_str(); - - CHECK(StringLength(str) == 0); - } - - SUBCASE("Short append") - { - sb.Append("abcd"); - CHECK(sb.IsDynamic() == false); - } - - SUBCASE("Short+long append") - { - sb.Append("abcd"); - CHECK(sb.IsDynamic() == false); - // This should trigger a dynamic buffer allocation since the required - // capacity exceeds the internal fixed buffer. - sb.Append("abcdefghijklmnopqrstuvwxyz"); - CHECK(sb.IsDynamic() == true); - CHECK(sb.Size() == 30); - CHECK(sb.Size() == StringLength(sb.c_str())); - } -} - -TEST_CASE("WideStringBuilder") -{ - WideStringBuilder<64> sb; - - SUBCASE("Empty init") - { - const wchar_t* str = sb.c_str(); - - CHECK(StringLength(str) == 0); - } - - SUBCASE("Append single character") - { - sb.Append(L'a'); - - const wchar_t* str = sb.c_str(); - CHECK(StringLength(str) == 1); - CHECK(str[0] == L'a'); - - sb.Append(L'b'); - str = sb.c_str(); - CHECK(StringLength(str) == 2); - CHECK(str[0] == L'a'); - CHECK(str[1] == L'b'); - } - - SUBCASE("Append string") - { - sb.Append(L"a"); - - const wchar_t* str = sb.c_str(); - CHECK(StringLength(str) == 1); - CHECK(str[0] == L'a'); - - sb.Append(L"b"); - str = sb.c_str(); - CHECK(StringLength(str) == 2); - CHECK(str[0] == L'a'); - CHECK(str[1] == L'b'); - - sb.Append(L"cdefghijklmnopqrstuvwxyz"); - CHECK(sb.Size() == 26); - - sb.Append(L"abcdefghijklmnopqrstuvwxyz"); - CHECK(sb.Size() == 52); - - sb.Append(L"abcdefghijk"); - CHECK(sb.Size() == 63); - } -} - -TEST_CASE("ExtendableWideStringBuilder") -{ - ExtendableWideStringBuilder<16> sb; - - SUBCASE("Empty init") - { - CHECK(sb.Size() == 0); - - const wchar_t* str = sb.c_str(); - CHECK(StringLength(str) == 0); - } - - SUBCASE("Short append") - { - sb.Append(L"abcd"); - CHECK(sb.IsDynamic() == false); - } - - SUBCASE("Short+long append") - { - sb.Append(L"abcd"); - CHECK(sb.IsDynamic() == false); - // This should trigger a dynamic buffer allocation since the required - // capacity exceeds the internal fixed buffer. - sb.Append(L"abcdefghijklmnopqrstuvwxyz"); - CHECK(sb.IsDynamic() == true); - CHECK(sb.Size() == 30); - CHECK(sb.Size() == StringLength(sb.c_str())); - } -} - -TEST_CASE("utf8") -{ - SUBCASE("utf8towide") - { - // TODO: add more extensive testing here - this covers a very small space - - WideStringBuilder<32> wout; - Utf8ToWide(u8"abcdefghi", wout); - CHECK(StringEquals(L"abcdefghi", wout.c_str())); - - wout.Reset(); - - Utf8ToWide(u8"abc���", wout); - CHECK(StringEquals(L"abc���", wout.c_str())); - } - - SUBCASE("widetoutf8") - { - // TODO: add more extensive testing here - this covers a very small space - - StringBuilder<32> out; - - WideToUtf8(L"abcdefghi", out); - CHECK(StringEquals("abcdefghi", out.c_str())); - - out.Reset(); - - WideToUtf8(L"abc���", out); - CHECK(StringEquals(u8"abc���", out.c_str())); - } -} - -TEST_CASE("filepath") -{ - CHECK(FilepathFindExtension("foo\\bar\\baz.txt", ".txt") != nullptr); - CHECK(FilepathFindExtension("foo\\bar\\baz.txt", ".zap") == nullptr); - - CHECK(FilepathFindExtension("foo\\bar\\baz.txt") != nullptr); - CHECK(FilepathFindExtension("foo\\bar\\baz.txt") == std::string_view(".txt")); - - CHECK(FilepathFindExtension(".txt") == std::string_view(".txt")); -} - -TEST_CASE("string") -{ - using namespace std::literals; - - SUBCASE("hash_djb2") - { - CHECK(HashStringAsLowerDjb2("AbcdZ"sv) == HashStringDjb2("abcdz"sv)); - CHECK(HashStringAsLowerDjb2("aBCd"sv) == HashStringDjb2("abcd"sv)); - CHECK(HashStringAsLowerDjb2("aBCd"sv) == HashStringDjb2(ToLower("aBCd"sv))); - } - - SUBCASE("tolower") - { - CHECK_EQ(ToLower("te!st"sv), "te!st"sv); - CHECK_EQ(ToLower("TE%St"sv), "te%st"sv); - } - - SUBCASE("StrCaseCompare") - { - CHECK(StrCaseCompare("foo", "FoO") == 0); - CHECK(StrCaseCompare("Bar", "bAs") < 0); - CHECK(StrCaseCompare("bAr", "Bas") < 0); - CHECK(StrCaseCompare("BBr", "Bar") > 0); - CHECK(StrCaseCompare("Bbr", "BAr") > 0); - CHECK(StrCaseCompare("foo", "FoO", 3) == 0); - CHECK(StrCaseCompare("Bar", "bAs", 3) < 0); - CHECK(StrCaseCompare("BBr", "Bar", 2) > 0); - } - - SUBCASE("ForEachStrTok") - { - const auto Tokens = "here,is,my,different,tokens"sv; - int32_t ExpectedTokenCount = 5; - int32_t TokenCount = 0; - StringBuilder<512> Sb; - - TokenCount = ForEachStrTok(Tokens, ',', [&Sb](const std::string_view& Token) { - if (Sb.Size()) - { - Sb << ","; - } - Sb << Token; - return true; - }); - - CHECK(TokenCount == ExpectedTokenCount); - CHECK(Sb.ToString() == Tokens); - - ExpectedTokenCount = 1; - const auto Str = "mosdef"sv; - - Sb.Reset(); - TokenCount = ForEachStrTok(Str, ' ', [&Sb](const std::string_view& Token) { - Sb << Token; - return true; - }); - CHECK(Sb.ToString() == Str); - CHECK(TokenCount == ExpectedTokenCount); - - ExpectedTokenCount = 0; - TokenCount = ForEachStrTok(""sv, ',', [](const std::string_view&) { return true; }); - CHECK(TokenCount == ExpectedTokenCount); - } -} - -#endif - -} // namespace zen diff --git a/zencore/testing.cpp b/zencore/testing.cpp deleted file mode 100644 index 1599e9d1f..000000000 --- a/zencore/testing.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/testing.h" -#include "zencore/logging.h" - -#if ZEN_WITH_TESTS - -namespace zen::testing { - -using namespace std::literals; - -struct TestRunner::Impl -{ - doctest::Context Session; -}; - -TestRunner::TestRunner() -{ - m_Impl = std::make_unique<Impl>(); -} - -TestRunner::~TestRunner() -{ -} - -int -TestRunner::ApplyCommandLine(int argc, char const* const* argv) -{ - m_Impl->Session.applyCommandLine(argc, argv); - - for (int i = 1; i < argc; ++i) - { - if (argv[i] == "--debug"sv) - { - spdlog::set_level(spdlog::level::debug); - } - } - - return 0; -} - -int -TestRunner::Run() -{ - int Rv = 0; - - m_Impl->Session.run(); - - return Rv; -} - -} // namespace zen::testing - -#endif diff --git a/zencore/testutils.cpp b/zencore/testutils.cpp deleted file mode 100644 index dbc3ab5af..000000000 --- a/zencore/testutils.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencore/testutils.h" -#include <zencore/session.h> -#include "zencore/string.h" - -#include <atomic> - -namespace zen { - -static std::atomic<int> Sequence{0}; - -std::filesystem::path -CreateTemporaryDirectory() -{ - std::error_code Ec; - - std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str(); - std::filesystem::remove_all(DirPath, Ec); - std::filesystem::create_directories(DirPath); - - return DirPath; -} - -ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporaryDirectory()) -{ -} - -ScopedTemporaryDirectory::ScopedTemporaryDirectory(std::filesystem::path Directory) : m_RootPath(Directory) -{ - std::error_code Ec; - std::filesystem::remove_all(Directory, Ec); - std::filesystem::create_directories(Directory); -} - -ScopedTemporaryDirectory::~ScopedTemporaryDirectory() -{ - std::error_code Ec; - std::filesystem::remove_all(m_RootPath, Ec); -} - -} // namespace zen
\ No newline at end of file diff --git a/zencore/thread.cpp b/zencore/thread.cpp deleted file mode 100644 index 1597a7dd9..000000000 --- a/zencore/thread.cpp +++ /dev/null @@ -1,1212 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/thread.h> - -#include <zencore/except.h> -#include <zencore/filesystem.h> -#include <zencore/scopeguard.h> -#include <zencore/string.h> -#include <zencore/testing.h> - -#if ZEN_PLATFORM_LINUX -# if !defined(_GNU_SOURCE) -# define _GNU_SOURCE // for semtimedop() -# endif -#endif - -#if ZEN_PLATFORM_WINDOWS -# include <shellapi.h> -# include <Shlobj.h> -# include <zencore/windows.h> -#else -# include <chrono> -# include <condition_variable> -# include <mutex> - -# include <fcntl.h> -# include <pthread.h> -# include <signal.h> -# include <sys/file.h> -# include <sys/sem.h> -# include <sys/stat.h> -# include <sys/syscall.h> -# include <sys/wait.h> -# include <time.h> -# include <unistd.h> -#endif - -#include <thread> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <fmt/format.h> -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -#if ZEN_PLATFORM_WINDOWS -// The information on how to set the thread name comes from -// a MSDN article: http://msdn2.microsoft.com/en-us/library/xcb2z8hs.aspx -const DWORD kVCThreadNameException = 0x406D1388; -typedef struct tagTHREADNAME_INFO -{ - DWORD dwType; // Must be 0x1000. - LPCSTR szName; // Pointer to name (in user addr space). - DWORD dwThreadID; // Thread ID (-1=caller thread). - DWORD dwFlags; // Reserved for future use, must be zero. -} THREADNAME_INFO; -// The SetThreadDescription API was brought in version 1607 of Windows 10. -typedef HRESULT(WINAPI* SetThreadDescription)(HANDLE hThread, PCWSTR lpThreadDescription); -// This function has try handling, so it is separated out of its caller. -void -SetNameInternal(DWORD thread_id, const char* name) -{ - THREADNAME_INFO info; - info.dwType = 0x1000; - info.szName = name; - info.dwThreadID = thread_id; - info.dwFlags = 0; - __try - { - RaiseException(kVCThreadNameException, 0, sizeof(info) / sizeof(DWORD), reinterpret_cast<DWORD_PTR*>(&info)); - } - __except (EXCEPTION_CONTINUE_EXECUTION) - { - } -} -#endif - -#if ZEN_PLATFORM_LINUX -const bool bNoZombieChildren = []() { - // When a child process exits it is put into a zombie state until the parent - // collects its result. This doesn't fit the Windows-like model that Zen uses - // where there is a less strict familial model and no zombification. Ignoring - // SIGCHLD siganals removes the need to call wait() on zombies. Another option - // would be for the child to call setsid() but that would detatch the child - // from the terminal. - struct sigaction Action = {}; - sigemptyset(&Action.sa_mask); - Action.sa_handler = SIG_IGN; - sigaction(SIGCHLD, &Action, nullptr); - return true; -}(); -#endif - -void -SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName) -{ -#if ZEN_PLATFORM_WINDOWS - // The SetThreadDescription API works even if no debugger is attached. - static auto SetThreadDescriptionFunc = - reinterpret_cast<SetThreadDescription>(::GetProcAddress(::GetModuleHandle(L"Kernel32.dll"), "SetThreadDescription")); - - if (SetThreadDescriptionFunc) - { - SetThreadDescriptionFunc(::GetCurrentThread(), Utf8ToWide(ThreadName).c_str()); - } - // The debugger needs to be around to catch the name in the exception. If - // there isn't a debugger, we are just needlessly throwing an exception. - if (!::IsDebuggerPresent()) - return; - - std::string ThreadNameZ{ThreadName}; - SetNameInternal(GetCurrentThreadId(), ThreadNameZ.c_str()); -#else - std::string ThreadNameZ{ThreadName}; -# if ZEN_PLATFORM_MAC - pthread_setname_np(ThreadNameZ.c_str()); -# else - pthread_setname_np(pthread_self(), ThreadNameZ.c_str()); -# endif -#endif -} // namespace zen - -void -RwLock::AcquireShared() -{ - m_Mutex.lock_shared(); -} - -void -RwLock::ReleaseShared() -{ - m_Mutex.unlock_shared(); -} - -void -RwLock::AcquireExclusive() -{ - m_Mutex.lock(); -} - -void -RwLock::ReleaseExclusive() -{ - m_Mutex.unlock(); -} - -////////////////////////////////////////////////////////////////////////// - -#if !ZEN_PLATFORM_WINDOWS -struct EventInner -{ - std::mutex Mutex; - std::condition_variable CondVar; - bool volatile bSet = false; -}; -#endif // !ZEN_PLATFORM_WINDOWS - -Event::Event() -{ - bool bManualReset = true; - bool bInitialState = false; - -#if ZEN_PLATFORM_WINDOWS - m_EventHandle = CreateEvent(nullptr, bManualReset, bInitialState, nullptr); -#else - ZEN_UNUSED(bManualReset); - auto* Inner = new EventInner(); - Inner->bSet = bInitialState; - m_EventHandle = Inner; -#endif -} - -Event::~Event() -{ - Close(); -} - -void -Event::Set() -{ -#if ZEN_PLATFORM_WINDOWS - SetEvent(m_EventHandle); -#else - auto* Inner = (EventInner*)m_EventHandle; - { - std::unique_lock Lock(Inner->Mutex); - Inner->bSet = true; - } - Inner->CondVar.notify_all(); -#endif -} - -void -Event::Reset() -{ -#if ZEN_PLATFORM_WINDOWS - ResetEvent(m_EventHandle); -#else - auto* Inner = (EventInner*)m_EventHandle; - { - std::unique_lock Lock(Inner->Mutex); - Inner->bSet = false; - } -#endif -} - -void -Event::Close() -{ -#if ZEN_PLATFORM_WINDOWS - CloseHandle(m_EventHandle); -#else - auto* Inner = (EventInner*)m_EventHandle; - delete Inner; -#endif - m_EventHandle = nullptr; -} - -bool -Event::Wait(int TimeoutMs) -{ -#if ZEN_PLATFORM_WINDOWS - using namespace std::literals; - - const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs; - - DWORD Result = WaitForSingleObject(m_EventHandle, Timeout); - - if (Result == WAIT_FAILED) - { - zen::ThrowLastError("Event wait failed"sv); - } - - return (Result == WAIT_OBJECT_0); -#else - auto* Inner = (EventInner*)m_EventHandle; - - if (TimeoutMs >= 0) - { - std::unique_lock Lock(Inner->Mutex); - - if (Inner->bSet) - { - return true; - } - - return Inner->CondVar.wait_for(Lock, std::chrono::milliseconds(TimeoutMs), [&] { return Inner->bSet; }); - } - - std::unique_lock Lock(Inner->Mutex); - - if (!Inner->bSet) - { - Inner->CondVar.wait(Lock, [&] { return Inner->bSet; }); - } - - return true; -#endif -} - -////////////////////////////////////////////////////////////////////////// - -NamedEvent::NamedEvent(std::string_view EventName) -{ -#if ZEN_PLATFORM_WINDOWS - using namespace std::literals; - - ExtendableStringBuilder<64> Name; - Name << "Local\\"sv; - Name << EventName; - - m_EventHandle = CreateEventA(nullptr, true, false, Name.c_str()); -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - // Create a file to back the semaphore - ExtendableStringBuilder<64> EventPath; - EventPath << "/tmp/" << EventName; - - int Fd = open(EventPath.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); - if (Fd < 0) - { - ThrowLastError(fmt::format("Failed to create '{}' for named event", EventPath)); - } - fchmod(Fd, 0666); - - // Use the file path to generate an IPC key - key_t IpcKey = ftok(EventPath.c_str(), 1); - if (IpcKey < 0) - { - close(Fd); - ThrowLastError("Failed to create an SysV IPC key"); - } - - // Use the key to create/open the semaphore - int Sem = semget(IpcKey, 1, 0600 | IPC_CREAT); - if (Sem < 0) - { - close(Fd); - ThrowLastError("Failed creating an SysV semaphore"); - } - - // Atomically claim ownership of the semaphore's key. The owner initialises - // the semaphore to 1 so we can use the wait-for-zero op as that does not - // modify the semaphore's value on a successful wait. - int LockResult = flock(Fd, LOCK_EX | LOCK_NB); - if (LockResult == 0) - { - // This isn't thread safe really. Another thread could open the same - // semaphore and successfully wait on it in the period of time where - // this comment is but before the semaphore's initialised. - semctl(Sem, 0, SETVAL, 1); - } - - // Pack into the handle - static_assert(sizeof(Sem) + sizeof(Fd) <= sizeof(void*), "Semaphore packing assumptions not met"); - intptr_t Packed; - Packed = intptr_t(Sem) << 32; - Packed |= intptr_t(Fd) & 0xffff'ffff; - m_EventHandle = (void*)Packed; -#endif -} - -NamedEvent::~NamedEvent() -{ - Close(); -} - -void -NamedEvent::Close() -{ - if (m_EventHandle == nullptr) - { - return; - } - -#if ZEN_PLATFORM_WINDOWS - CloseHandle(m_EventHandle); -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Fd = int(intptr_t(m_EventHandle) & 0xffff'ffff); - - if (flock(Fd, LOCK_EX | LOCK_NB) == 0) - { - std::filesystem::path Name = PathFromHandle((void*)(intptr_t(Fd))); - unlink(Name.c_str()); - - flock(Fd, LOCK_UN | LOCK_NB); - close(Fd); - - int Sem = int(intptr_t(m_EventHandle) >> 32); - semctl(Sem, 0, IPC_RMID); - } -#endif - - m_EventHandle = nullptr; -} - -void -NamedEvent::Set() -{ -#if ZEN_PLATFORM_WINDOWS - SetEvent(m_EventHandle); -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Sem = int(intptr_t(m_EventHandle) >> 32); - semctl(Sem, 0, SETVAL, 0); -#endif -} - -bool -NamedEvent::Wait(int TimeoutMs) -{ -#if ZEN_PLATFORM_WINDOWS - const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs; - - DWORD Result = WaitForSingleObject(m_EventHandle, Timeout); - - if (Result == WAIT_FAILED) - { - using namespace std::literals; - zen::ThrowLastError("Event wait failed"sv); - } - - return (Result == WAIT_OBJECT_0); -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Sem = int(intptr_t(m_EventHandle) >> 32); - - int Result; - struct sembuf SemOp = {}; - - if (TimeoutMs < 0) - { - Result = semop(Sem, &SemOp, 1); - return Result == 0; - } - -# if defined(_GNU_SOURCE) - struct timespec TimeoutValue = { - .tv_sec = TimeoutMs >> 10, - .tv_nsec = (TimeoutMs & 0x3ff) << 20, - }; - Result = semtimedop(Sem, &SemOp, 1, &TimeoutValue); -# else - const int SleepTimeMs = 10; - SemOp.sem_flg = IPC_NOWAIT; - do - { - Result = semop(Sem, &SemOp, 1); - if (Result == 0 || errno != EAGAIN) - { - break; - } - - Sleep(SleepTimeMs); - TimeoutMs -= SleepTimeMs; - } while (TimeoutMs > 0); -# endif // _GNU_SOURCE - - return Result == 0; -#endif -} - -////////////////////////////////////////////////////////////////////////// - -NamedMutex::~NamedMutex() -{ -#if ZEN_PLATFORM_WINDOWS - if (m_MutexHandle) - { - CloseHandle(m_MutexHandle); - } -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Inner = int(intptr_t(m_MutexHandle)); - flock(Inner, LOCK_UN); - close(Inner); -#endif -} - -bool -NamedMutex::Create(std::string_view MutexName) -{ -#if ZEN_PLATFORM_WINDOWS - ZEN_ASSERT(m_MutexHandle == nullptr); - - using namespace std::literals; - - ExtendableStringBuilder<64> Name; - Name << "Global\\"sv; - Name << MutexName; - - m_MutexHandle = CreateMutexA(nullptr, /* InitialOwner */ TRUE, Name.c_str()); - - return !!m_MutexHandle; -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - ExtendableStringBuilder<64> Name; - Name << "/tmp/" << MutexName; - - int Inner = open(Name.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); - if (Inner < 0) - { - return false; - } - fchmod(Inner, 0666); - - if (flock(Inner, LOCK_EX) != 0) - { - close(Inner); - Inner = 0; - return false; - } - - m_MutexHandle = (void*)(intptr_t(Inner)); - return true; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -NamedMutex::Exists(std::string_view MutexName) -{ -#if ZEN_PLATFORM_WINDOWS - using namespace std::literals; - - ExtendableStringBuilder<64> Name; - Name << "Global\\"sv; - Name << MutexName; - - void* MutexHandle = OpenMutexA(SYNCHRONIZE, /* InheritHandle */ FALSE, Name.c_str()); - - if (MutexHandle == nullptr) - { - return false; - } - - CloseHandle(MutexHandle); - - return true; -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - ExtendableStringBuilder<64> Name; - Name << "/tmp/" << MutexName; - - bool bExists = false; - int Fd = open(Name.c_str(), O_RDWR | O_CLOEXEC); - if (Fd >= 0) - { - if (flock(Fd, LOCK_EX | LOCK_NB) == 0) - { - flock(Fd, LOCK_UN | LOCK_NB); - } - else - { - bExists = true; - } - close(Fd); - } - - return bExists; -#endif // ZEN_PLATFORM_WINDOWS -} - -////////////////////////////////////////////////////////////////////////// - -ProcessHandle::ProcessHandle() = default; - -#if ZEN_PLATFORM_WINDOWS -void -ProcessHandle::Initialize(void* ProcessHandle) -{ - ZEN_ASSERT(m_ProcessHandle == nullptr); - - if (ProcessHandle == INVALID_HANDLE_VALUE) - { - ProcessHandle = nullptr; - } - - // TODO: perform some debug verification here to verify it's a valid handle? - m_ProcessHandle = ProcessHandle; - m_Pid = GetProcessId(m_ProcessHandle); -} -#endif // ZEN_PLATFORM_WINDOWS - -ProcessHandle::~ProcessHandle() -{ - Reset(); -} - -void -ProcessHandle::Initialize(int Pid) -{ - ZEN_ASSERT(m_ProcessHandle == nullptr); - -#if ZEN_PLATFORM_WINDOWS - m_ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, Pid); -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - if (Pid > 0) - { - m_ProcessHandle = (void*)(intptr_t(Pid)); - } -#endif - - if (!m_ProcessHandle) - { - ThrowLastError(fmt::format("ProcessHandle::Initialize(pid: {}) failed", Pid)); - } - - m_Pid = Pid; -} - -bool -ProcessHandle::IsRunning() const -{ - bool bActive = false; - -#if ZEN_PLATFORM_WINDOWS - DWORD ExitCode = 0; - GetExitCodeProcess(m_ProcessHandle, &ExitCode); - bActive = (ExitCode == STILL_ACTIVE); -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - bActive = (kill(pid_t(m_Pid), 0) == 0); -#endif - - return bActive; -} - -bool -ProcessHandle::IsValid() const -{ - return (m_ProcessHandle != nullptr); -} - -void -ProcessHandle::Terminate(int ExitCode) -{ - if (!IsRunning()) - { - return; - } - - bool bSuccess = false; - -#if ZEN_PLATFORM_WINDOWS - TerminateProcess(m_ProcessHandle, ExitCode); - DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, INFINITE); - bSuccess = (WaitResult != WAIT_OBJECT_0); -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - ZEN_UNUSED(ExitCode); - bSuccess = (kill(m_Pid, SIGKILL) == 0); -#endif - - if (!bSuccess) - { - // What might go wrong here, and what is meaningful to act on? - } -} - -void -ProcessHandle::Reset() -{ - if (IsValid()) - { -#if ZEN_PLATFORM_WINDOWS - CloseHandle(m_ProcessHandle); -#endif - m_ProcessHandle = nullptr; - m_Pid = 0; - } -} - -bool -ProcessHandle::Wait(int TimeoutMs) -{ - using namespace std::literals; - -#if ZEN_PLATFORM_WINDOWS - const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs; - - const DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, Timeout); - - switch (WaitResult) - { - case WAIT_OBJECT_0: - return true; - - case WAIT_TIMEOUT: - return false; - - case WAIT_FAILED: - break; - } -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - const int SleepMs = 20; - timespec SleepTime = {0, SleepMs * 1000 * 1000}; - for (int i = 0;; i += SleepMs) - { -# if ZEN_PLATFORM_MAC - int WaitState = 0; - waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED); -# endif - - if (kill(m_Pid, 0) < 0) - { - if (zen::GetLastError() == ESRCH) - { - return true; - } - break; - } - - if (TimeoutMs >= 0 && i >= TimeoutMs) - { - return false; - } - - nanosleep(&SleepTime, nullptr); - } -#endif - - // What might go wrong here, and what is meaningful to act on? - ThrowLastError("Process::Wait failed"sv); -} - -////////////////////////////////////////////////////////////////////////// - -#if !ZEN_PLATFORM_WINDOWS || ZEN_WITH_TESTS -static void -BuildArgV(std::vector<char*>& Out, char* CommandLine) -{ - char* Cursor = CommandLine; - while (true) - { - // Skip leading whitespace - for (; *Cursor == ' '; ++Cursor) - ; - - // Check for nullp terminator - if (*Cursor == '\0') - { - break; - } - - Out.push_back(Cursor); - - // Extract word - int QuoteCount = 0; - do - { - QuoteCount += (*Cursor == '\"'); - if (*Cursor == ' ' && !(QuoteCount & 1)) - { - break; - } - ++Cursor; - } while (*Cursor != '\0'); - - if (*Cursor == '\0') - { - break; - } - - *Cursor = '\0'; - ++Cursor; - } -} -#endif // !WINDOWS || TESTS - -#if ZEN_PLATFORM_WINDOWS -static CreateProcResult -CreateProcNormal(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options) -{ - PROCESS_INFORMATION ProcessInfo{}; - STARTUPINFO StartupInfo{.cb = sizeof(STARTUPINFO)}; - - const bool InheritHandles = false; - void* Environment = nullptr; - LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr; - LPSECURITY_ATTRIBUTES ThreadAttributes = nullptr; - - DWORD CreationFlags = 0; - if (Options.Flags & CreateProcOptions::Flag_NewConsole) - { - CreationFlags |= CREATE_NEW_CONSOLE; - } - - const wchar_t* WorkingDir = nullptr; - if (Options.WorkingDirectory != nullptr) - { - WorkingDir = Options.WorkingDirectory->c_str(); - } - - ExtendableWideStringBuilder<256> CommandLineZ; - CommandLineZ << CommandLine; - - BOOL Success = CreateProcessW(Executable.c_str(), - CommandLineZ.Data(), - ProcessAttributes, - ThreadAttributes, - InheritHandles, - CreationFlags, - Environment, - WorkingDir, - &StartupInfo, - &ProcessInfo); - - if (!Success) - { - return nullptr; - } - - CloseHandle(ProcessInfo.hThread); - return ProcessInfo.hProcess; -} - -static CreateProcResult -CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options) -{ - /* Launches a binary with the shell as its parent. The shell (such as - Explorer) should be an unelevated process. */ - - // No sense in using this route if we are not elevated in the first place - if (IsUserAnAdmin() == FALSE) - { - return CreateProcNormal(Executable, CommandLine, Options); - } - - // Get the users' shell process and open it for process creation - HWND ShellWnd = GetShellWindow(); - if (ShellWnd == nullptr) - { - return nullptr; - } - - DWORD ShellPid; - GetWindowThreadProcessId(ShellWnd, &ShellPid); - - HANDLE Process = OpenProcess(PROCESS_CREATE_PROCESS, FALSE, ShellPid); - if (Process == nullptr) - { - return nullptr; - } - auto $0 = MakeGuard([&] { CloseHandle(Process); }); - - // Creating a process as a child of another process is done by setting a - // thread-attribute list on the startup info passed to CreateProcess() - SIZE_T AttrListSize; - InitializeProcThreadAttributeList(nullptr, 1, 0, &AttrListSize); - - auto AttrList = (PPROC_THREAD_ATTRIBUTE_LIST)malloc(AttrListSize); - auto $1 = MakeGuard([&] { free(AttrList); }); - - if (!InitializeProcThreadAttributeList(AttrList, 1, 0, &AttrListSize)) - { - return nullptr; - } - - BOOL bOk = - UpdateProcThreadAttribute(AttrList, 0, PROC_THREAD_ATTRIBUTE_PARENT_PROCESS, (HANDLE*)&Process, sizeof(Process), nullptr, nullptr); - if (!bOk) - { - return nullptr; - } - - // By this point we know we are an elevated process. It is not allowed to - // create a process as a child of another unelevated process that share our - // elevated console window if we have one. So we'll need to create a new one. - uint32_t CreateProcFlags = EXTENDED_STARTUPINFO_PRESENT; - if (GetConsoleWindow() != nullptr) - { - CreateProcFlags |= CREATE_NEW_CONSOLE; - } - else - { - CreateProcFlags |= DETACHED_PROCESS; - } - - // Everything is set up now so we can proceed and launch the process - STARTUPINFOEXW StartupInfo = { - .StartupInfo = {.cb = sizeof(STARTUPINFOEXW)}, - .lpAttributeList = AttrList, - }; - PROCESS_INFORMATION ProcessInfo = {}; - - if (Options.Flags & CreateProcOptions::Flag_NewConsole) - { - CreateProcFlags |= CREATE_NEW_CONSOLE; - } - - ExtendableWideStringBuilder<256> CommandLineZ; - CommandLineZ << CommandLine; - - bOk = CreateProcessW(Executable.c_str(), - CommandLineZ.Data(), - nullptr, - nullptr, - FALSE, - CreateProcFlags, - nullptr, - nullptr, - &StartupInfo.StartupInfo, - &ProcessInfo); - if (bOk == FALSE) - { - return nullptr; - } - - CloseHandle(ProcessInfo.hThread); - return ProcessInfo.hProcess; -} - -static CreateProcResult -CreateProcElevated(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options) -{ - ExtendableWideStringBuilder<256> CommandLineZ; - CommandLineZ << CommandLine; - - SHELLEXECUTEINFO ShellExecuteInfo; - ZeroMemory(&ShellExecuteInfo, sizeof(ShellExecuteInfo)); - ShellExecuteInfo.cbSize = sizeof(ShellExecuteInfo); - ShellExecuteInfo.fMask = SEE_MASK_UNICODE | SEE_MASK_NOCLOSEPROCESS; - ShellExecuteInfo.lpFile = Executable.c_str(); - ShellExecuteInfo.lpVerb = TEXT("runas"); - ShellExecuteInfo.nShow = SW_SHOW; - ShellExecuteInfo.lpParameters = CommandLineZ.c_str(); - - if (Options.WorkingDirectory != nullptr) - { - ShellExecuteInfo.lpDirectory = Options.WorkingDirectory->c_str(); - } - - if (::ShellExecuteEx(&ShellExecuteInfo)) - { - return ShellExecuteInfo.hProcess; - } - - return nullptr; -} -#endif // ZEN_PLATFORM_WINDOWS - -CreateProcResult -CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options) -{ -#if ZEN_PLATFORM_WINDOWS - if (Options.Flags & CreateProcOptions::Flag_Unelevated) - { - return CreateProcUnelevated(Executable, CommandLine, Options); - } - - if (Options.Flags & CreateProcOptions::Flag_Elevated) - { - return CreateProcElevated(Executable, CommandLine, Options); - } - - return CreateProcNormal(Executable, CommandLine, Options); -#else - std::vector<char*> ArgV; - std::string CommandLineZ(CommandLine); - BuildArgV(ArgV, CommandLineZ.data()); - ArgV.push_back(nullptr); - - int ChildPid = fork(); - if (ChildPid < 0) - { - ThrowLastError("Failed to fork a new child process"); - } - else if (ChildPid == 0) - { - if (Options.WorkingDirectory != nullptr) - { - int Result = chdir(Options.WorkingDirectory->c_str()); - ZEN_UNUSED(Result); - } - - if (execv(Executable.c_str(), ArgV.data()) < 0) - { - ThrowLastError("Failed to exec() a new process image"); - } - } - - return ChildPid; -#endif -} - -////////////////////////////////////////////////////////////////////////// - -ProcessMonitor::ProcessMonitor() -{ -} - -ProcessMonitor::~ProcessMonitor() -{ - RwLock::ExclusiveLockScope _(m_Lock); - - for (HandleType& Proc : m_ProcessHandles) - { -#if ZEN_PLATFORM_WINDOWS - CloseHandle(Proc); -#endif - Proc = 0; - } -} - -bool -ProcessMonitor::IsRunning() -{ - RwLock::ExclusiveLockScope _(m_Lock); - - bool FoundOne = false; - - for (HandleType& Proc : m_ProcessHandles) - { - bool ProcIsActive; - -#if ZEN_PLATFORM_WINDOWS - DWORD ExitCode = 0; - GetExitCodeProcess(Proc, &ExitCode); - - ProcIsActive = (ExitCode == STILL_ACTIVE); - if (!ProcIsActive) - { - CloseHandle(Proc); - } -#else - int Pid = int(intptr_t(Proc)); - ProcIsActive = IsProcessRunning(Pid); -#endif - - if (!ProcIsActive) - { - Proc = 0; - } - - // Still alive - FoundOne |= ProcIsActive; - } - - std::erase_if(m_ProcessHandles, [](HandleType Handle) { return Handle == 0; }); - - return FoundOne; -} - -void -ProcessMonitor::AddPid(int Pid) -{ - HandleType ProcessHandle; - -#if ZEN_PLATFORM_WINDOWS - ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, Pid); -#else - ProcessHandle = HandleType(intptr_t(Pid)); -#endif - - if (ProcessHandle) - { - RwLock::ExclusiveLockScope _(m_Lock); - m_ProcessHandles.push_back(ProcessHandle); - } -} - -bool -ProcessMonitor::IsActive() const -{ - RwLock::SharedLockScope _(m_Lock); - return m_ProcessHandles.empty() == false; -} - -////////////////////////////////////////////////////////////////////////// - -bool -IsProcessRunning(int pid) -{ - // This function is arguably not super useful, a pid can be re-used - // by the OS so holding on to a pid and polling it over some time - // period will not necessarily tell you what you probably want to know. - -#if ZEN_PLATFORM_WINDOWS - HANDLE hProc = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); - - if (!hProc) - { - DWORD Error = zen::GetLastError(); - - if (Error == ERROR_INVALID_PARAMETER) - { - return false; - } - - ThrowSystemError(Error, fmt::format("failed to open process with pid {}", pid)); - } - - CloseHandle(hProc); - - return true; -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - return (kill(pid_t(pid), 0) == 0); -#endif -} - -int -GetCurrentProcessId() -{ -#if ZEN_PLATFORM_WINDOWS - return ::GetCurrentProcessId(); -#else - return int(getpid()); -#endif -} - -int -GetCurrentThreadId() -{ -#if ZEN_PLATFORM_WINDOWS - return ::GetCurrentThreadId(); -#elif ZEN_PLATFORM_LINUX - return int(syscall(SYS_gettid)); -#elif ZEN_PLATFORM_MAC - return int(pthread_mach_thread_np(pthread_self())); -#endif -} - -void -Sleep(int ms) -{ -#if ZEN_PLATFORM_WINDOWS - ::Sleep(ms); -#else - usleep(ms * 1000U); -#endif -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -void -thread_forcelink() -{ -} - -TEST_CASE("Thread") -{ - int Pid = GetCurrentProcessId(); - CHECK(Pid > 0); - CHECK(IsProcessRunning(Pid)); - - CHECK_FALSE(GetCurrentThreadId() == 0); -} - -TEST_CASE("BuildArgV") -{ - const char* Words[] = {"one", "two", "three", "four", "five"}; - struct - { - int WordCount; - const char* Input; - } Cases[] = { - {0, ""}, - {0, " "}, - {1, "one"}, - {1, " one"}, - {1, "one "}, - {2, "one two"}, - {2, " one two"}, - {2, "one two "}, - {2, " one two"}, - {2, "one two "}, - {2, "one two "}, - {3, "one two three"}, - {3, "\"one\" two \"three\""}, - {5, "one two three four five"}, - }; - - for (const auto& Case : Cases) - { - std::vector<char*> OutArgs; - StringBuilder<64> Mutable; - Mutable << Case.Input; - BuildArgV(OutArgs, Mutable.Data()); - - CHECK_EQ(OutArgs.size(), Case.WordCount); - - for (int i = 0, n = int(OutArgs.size()); i < n; ++i) - { - const char* Truth = Words[i]; - size_t TruthLen = strlen(Truth); - - const char* Candidate = OutArgs[i]; - bool bQuoted = (Candidate[0] == '\"'); - Candidate += bQuoted; - - CHECK(strncmp(Truth, Candidate, TruthLen) == 0); - - if (bQuoted) - { - CHECK_EQ(Candidate[TruthLen], '\"'); - } - } - } -} - -TEST_CASE("NamedEvent") -{ - std::string Name = "zencore_test_event"; - NamedEvent TestEvent(Name); - - // Timeout test - for (uint32_t i = 0; i < 8; ++i) - { - bool bEventSet = TestEvent.Wait(100); - CHECK(!bEventSet); - } - - // Thread check - std::thread Waiter = std::thread([Name]() { - NamedEvent ReadyEvent(Name + "_ready"); - ReadyEvent.Set(); - - NamedEvent TestEvent(Name); - TestEvent.Wait(1000); - }); - - NamedEvent ReadyEvent(Name + "_ready"); - ReadyEvent.Wait(); - - zen::Sleep(500); - TestEvent.Set(); - - Waiter.join(); - - // Manual reset property - for (uint32_t i = 0; i < 8; ++i) - { - bool bEventSet = TestEvent.Wait(100); - CHECK(bEventSet); - } -} - -TEST_CASE("NamedMutex") -{ - static const char* Name = "zen_test_mutex"; - - CHECK(!NamedMutex::Exists(Name)); - - { - NamedMutex TestMutex; - CHECK(TestMutex.Create(Name)); - CHECK(NamedMutex::Exists(Name)); - } - - CHECK(!NamedMutex::Exists(Name)); -} - -#endif // ZEN_WITH_TESTS - -} // namespace zen diff --git a/zencore/timer.cpp b/zencore/timer.cpp deleted file mode 100644 index 1655e912d..000000000 --- a/zencore/timer.cpp +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/thread.h> -#include <zencore/timer.h> - -#include <zencore/testing.h> - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#elif ZEN_PLATFORM_LINUX -# include <time.h> -# include <unistd.h> -#endif - -namespace zen { - -uint64_t -GetHifreqTimerValue() -{ - uint64_t Timestamp; - -#if ZEN_PLATFORM_WINDOWS - LARGE_INTEGER li; - QueryPerformanceCounter(&li); - - Timestamp = li.QuadPart; -#else - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - Timestamp = (uint64_t(ts.tv_sec) * 1000000ull) + (uint64_t(ts.tv_nsec) / 1000ull); -#endif - - return Timestamp; -} - -uint64_t -InternalGetHifreqTimerFrequency() -{ -#if ZEN_PLATFORM_WINDOWS - LARGE_INTEGER li; - QueryPerformanceFrequency(&li); - - return li.QuadPart; -#else - return 1000000ull; -#endif -} - -uint64_t QpcFreq = InternalGetHifreqTimerFrequency(); -static const double QpcFactor = 1.0 / InternalGetHifreqTimerFrequency(); - -uint64_t -GetHifreqTimerFrequency() -{ - return QpcFreq; -} - -double -GetHifreqTimerToSeconds() -{ - return QpcFactor; -} - -uint64_t -GetHifreqTimerFrequencySafe() -{ - if (!QpcFreq) - { - QpcFreq = InternalGetHifreqTimerFrequency(); - } - - return QpcFreq; -} - -////////////////////////////////////////////////////////////////////////// - -uint64_t detail::g_LofreqTimerValue = GetHifreqTimerValue(); - -void -UpdateLofreqTimerValue() -{ - detail::g_LofreqTimerValue = GetHifreqTimerValue(); -} - -uint64_t -GetLofreqTimerFrequency() -{ - return GetHifreqTimerFrequencySafe(); -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -void -timer_forcelink() -{ -} - -#endif - -} // namespace zen diff --git a/zencore/trace.cpp b/zencore/trace.cpp deleted file mode 100644 index 788dcec07..000000000 --- a/zencore/trace.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -/* clang-format off */ - -#if ZEN_WITH_TRACE - -#include <zencore/zencore.h> - -#define TRACE_IMPLEMENT 1 -#include <zencore/trace.h> - -void -TraceInit(const char* HostOrPath, TraceType Type) -{ - bool EnableEvents = true; - - switch (Type) - { - case TraceType::Network: - trace::SendTo(HostOrPath); - break; - - case TraceType::File: - trace::WriteTo(HostOrPath); - break; - - case TraceType::None: - EnableEvents = false; - break; - } - - trace::FInitializeDesc Desc = { - .bUseImportantCache = false, - }; - trace::Initialize(Desc); - - if (EnableEvents) - { - trace::ToggleChannel("cpu", true); - } -} - -#endif // ZEN_WITH_TRACE - -/* clang-format on */ diff --git a/zencore/uid.cpp b/zencore/uid.cpp deleted file mode 100644 index 86cdfae3a..000000000 --- a/zencore/uid.cpp +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/uid.h> - -#include <zencore/endian.h> -#include <zencore/string.h> -#include <zencore/testing.h> - -#include <atomic> -#include <bit> -#include <chrono> -#include <random> -#include <set> -#include <unordered_map> - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -namespace detail { - static bool OidInitialised; - static uint32_t RunId; - static std::atomic_uint32_t Serial; -} // namespace detail - -////////////////////////////////////////////////////////////////////////// - -const Oid Oid::Zero = {{0u, 0u, 0u}}; -const Oid Oid::Max = {{~0u, ~0u, ~0u}}; - -void -Oid::Initialize() -{ - if (!detail::OidInitialised) - { - std::random_device Rng; - detail::RunId = Rng(); - detail::Serial = Rng(); - - detail::OidInitialised = true; - } -} - -const Oid& -Oid::Generate() -{ - if (!detail::OidInitialised) - { - Oid::Initialize(); - } - - const uint64_t kOffset = 1'609'459'200; // Seconds from 1970 -> 2021 - const uint64_t Time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) - kOffset; - - OidBits[0] = ToNetworkOrder(uint32_t(Time)); - OidBits[1] = ToNetworkOrder(uint32_t(++detail::Serial)); - OidBits[2] = detail::RunId; - - return *this; -} - -Oid -Oid::NewOid() -{ - return Oid().Generate(); -} - -Oid -Oid::FromHexString(const std::string_view String) -{ - ZEN_ASSERT(String.size() == 2 * sizeof(Oid::OidBits)); - - Oid Id; - - if (ParseHexBytes(String.data(), String.size(), reinterpret_cast<uint8_t*>(Id.OidBits))) - { - return Id; - } - else - { - return Oid::Zero; - } -} - -Oid -Oid::FromMemory(const void* Ptr) -{ - Oid Id; - memcpy(Id.OidBits, Ptr, sizeof Id); - return Id; -} - -void -Oid::ToString(char OutString[StringLength]) -{ - ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), OutString); -} - -StringBuilderBase& -Oid::ToString(StringBuilderBase& OutString) const -{ - String_t Str; - ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), Str); - - OutString.AppendRange(Str, &Str[StringLength]); - - return OutString; -} - -#if ZEN_WITH_TESTS - -TEST_CASE("Oid") -{ - SUBCASE("Basic") - { - Oid id1 = Oid::NewOid(); - ZEN_UNUSED(id1); - - std::vector<Oid> ids; - std::set<Oid> idset; - std::unordered_map<Oid, int, Oid::Hasher> idmap; - - const int Count = 1000; - - for (int i = 0; i < Count; ++i) - { - Oid id; - id.Generate(); - - ids.emplace_back(id); - idset.insert(id); - idmap.insert({id, i}); - } - - CHECK(ids.size() == Count); - CHECK(idset.size() == Count); // All ids should be unique - CHECK(idmap.size() == Count); // Ditto - } -} - -void -uid_forcelink() -{ -} - -#endif - -} // namespace zen diff --git a/zencore/workthreadpool.cpp b/zencore/workthreadpool.cpp deleted file mode 100644 index b4328cdbd..000000000 --- a/zencore/workthreadpool.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/workthreadpool.h> - -#include <zencore/logging.h> - -namespace zen { - -namespace detail { - struct LambdaWork : IWork - { - LambdaWork(auto Work) : WorkFunction(Work) {} - virtual void Execute() override { WorkFunction(); } - - std::function<void()> WorkFunction; - }; -} // namespace detail - -WorkerThreadPool::WorkerThreadPool(int InThreadCount) -{ - for (int i = 0; i < InThreadCount; ++i) - { - m_WorkerThreads.emplace_back(&WorkerThreadPool::WorkerThreadFunction, this); - } -} - -WorkerThreadPool::~WorkerThreadPool() -{ - m_WorkQueue.CompleteAdding(); - - for (std::thread& Thread : m_WorkerThreads) - { - Thread.join(); - } - - m_WorkerThreads.clear(); -} - -void -WorkerThreadPool::ScheduleWork(Ref<IWork> Work) -{ - m_WorkQueue.Enqueue(std::move(Work)); -} - -void -WorkerThreadPool::ScheduleWork(std::function<void()>&& Work) -{ - m_WorkQueue.Enqueue(Ref<IWork>(new detail::LambdaWork(Work))); -} - -[[nodiscard]] size_t -WorkerThreadPool::PendingWork() const -{ - return m_WorkQueue.Size(); -} - -void -WorkerThreadPool::WorkerThreadFunction() -{ - do - { - Ref<IWork> Work; - if (m_WorkQueue.WaitAndDequeue(Work)) - { - try - { - Work->Execute(); - } - catch (std::exception& e) - { - Work->m_Exception = std::current_exception(); - - ZEN_WARN("Caught exception in worker thread: {}", e.what()); - } - } - else - { - return; - } - } while (true); -} - -} // namespace zen diff --git a/zencore/xmake.lua b/zencore/xmake.lua deleted file mode 100644 index e1e649c1d..000000000 --- a/zencore/xmake.lua +++ /dev/null @@ -1,61 +0,0 @@ --- Copyright Epic Games, Inc. All Rights Reserved. - -target('zencore') - set_kind("static") - add_headerfiles("**.h") - add_configfiles("include/zencore/config.h.in") - on_load(function (target) - local version = io.readfile("VERSION.txt") - version = string.gsub(version,"%-pre.*", "") - target:set("version", version:trim(), {build = "%Y%m%d%H%M"}) - end) - set_configdir("include/zencore") - add_files("**.cpp") - add_includedirs("include", {public=true}) - add_includedirs("$(projectdir)/thirdparty/utfcpp/source") - add_includedirs("$(projectdir)/thirdparty/trace", {public=true}) - if is_os("windows") then - add_linkdirs("$(projectdir)/thirdparty/Oodle/lib/Win64") - elseif is_os("linux") then - add_linkdirs("$(projectdir)/thirdparty/Oodle/lib/Linux_x64") - add_links("oo2corelinux64") - add_syslinks("pthread") - elseif is_os("macosx") then - add_linkdirs("$(projectdir)/thirdparty/Oodle/lib/Mac_x64") - add_links("oo2coremac64") - end - add_options("zentrace") - add_packages( - "vcpkg::blake3", - "vcpkg::cpr", - "vcpkg::curl", -- required by cpr - "vcpkg::doctest", - "vcpkg::fmt", - "vcpkg::gsl-lite", - "vcpkg::json11", - "vcpkg::lz4", - "vcpkg::mimalloc", - "vcpkg::openssl", -- required by curl - "vcpkg::spdlog", - "vcpkg::zlib", -- required by curl - "vcpkg::xxhash") - - if is_plat("linux") then - -- The 'vcpkg::openssl' package is two libraries; ssl and crypto, with - -- ssl being dependent on symbols in crypto. When GCC-like linkers read - -- object files from their command line, those object files only resolve - -- symbols of objects previously encountered. Thus crypto must appear - -- after ssl so it can fill out ssl's unresolved symbol table. Xmake's - -- vcpkg support is basic and works by parsing .list files. Openssl's - -- archives are listed alphabetically causing crypto to be _before_ ssl - -- and resulting in link errors. The links are restated here to force - -- xmake to use the correct order, and "syslinks" is used to force the - -- arguments to the end of the line (otherwise they can appear before - -- curl and cause more errors). - add_syslinks("crypto") - add_syslinks("dl") - end - - if is_plat("linux") then - add_syslinks("rt") - end diff --git a/zencore/xxhash.cpp b/zencore/xxhash.cpp deleted file mode 100644 index 450131d19..000000000 --- a/zencore/xxhash.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/xxhash.h> - -#include <zencore/string.h> -#include <zencore/testing.h> - -#include <gsl/gsl-lite.hpp> - -namespace zen { - -XXH3_128 XXH3_128::Zero; // Initialized to all zeros - -XXH3_128 -XXH3_128::FromHexString(const char* InString) -{ - return FromHexString({InString, sizeof(XXH3_128::Hash) * 2}); -} - -XXH3_128 -XXH3_128::FromHexString(std::string_view InString) -{ - ZEN_ASSERT(InString.size() == 2 * sizeof(XXH3_128::Hash)); - - XXH3_128 Xx; - ParseHexBytes(InString.data(), InString.size(), Xx.Hash); - return Xx; -} - -const char* -XXH3_128::ToHexString(char* OutString /* 40 characters + NUL terminator */) const -{ - ToHexBytes(Hash, sizeof(XXH3_128), OutString); - OutString[2 * sizeof(XXH3_128)] = '\0'; - - return OutString; -} - -StringBuilderBase& -XXH3_128::ToHexString(StringBuilderBase& OutBuilder) const -{ - String_t str; - ToHexString(str); - - OutBuilder.AppendRange(str, &str[StringLength]); - - return OutBuilder; -} - -} // namespace zen diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp deleted file mode 100644 index 2a7c5755e..000000000 --- a/zencore/zencore.cpp +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/zencore.h> - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#endif - -#if ZEN_PLATFORM_LINUX -# include <pthread.h> -#endif - -#include <zencore/blake3.h> -#include <zencore/compactbinary.h> -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/compositebuffer.h> -#include <zencore/compress.h> -#include <zencore/crypto.h> -#include <zencore/filesystem.h> -#include <zencore/intmath.h> -#include <zencore/iobuffer.h> -#include <zencore/memory.h> -#include <zencore/mpscqueue.h> -#include <zencore/refcount.h> -#include <zencore/sha1.h> -#include <zencore/stats.h> -#include <zencore/stream.h> -#include <zencore/string.h> -#include <zencore/thread.h> -#include <zencore/timer.h> -#include <zencore/uid.h> - -namespace zen { - -AssertImpl AssertImpl::DefaultAssertImpl; -AssertImpl* AssertImpl::CurrentAssertImpl = &AssertImpl::DefaultAssertImpl; - -////////////////////////////////////////////////////////////////////////// - -bool -IsDebuggerPresent() -{ -#if ZEN_PLATFORM_WINDOWS - return ::IsDebuggerPresent(); -#else - return false; -#endif -} - -std::optional<bool> InteractiveSessionFlag; - -void -SetIsInteractiveSession(bool Value) -{ - InteractiveSessionFlag = Value; -} - -bool -IsInteractiveSession() -{ - if (!InteractiveSessionFlag.has_value()) - { -#if ZEN_PLATFORM_WINDOWS - DWORD dwSessionId = 0; - if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) - { - InteractiveSessionFlag = (dwSessionId != 0); - } - else - { - InteractiveSessionFlag = false; - } -#else - // TODO: figure out what actually makes sense here - InteractiveSessionFlag = true; -#endif - } - - return InteractiveSessionFlag.value(); -} - -////////////////////////////////////////////////////////////////////////// - -static int s_ApplicationExitCode = 0; -static bool s_ApplicationExitRequested; - -bool -IsApplicationExitRequested() -{ - return s_ApplicationExitRequested; -} - -void -RequestApplicationExit(int ExitCode) -{ - s_ApplicationExitCode = ExitCode; - s_ApplicationExitRequested = true; -} - -#if ZEN_WITH_TESTS -void -zencore_forcelinktests() -{ - zen::blake3_forcelink(); - zen::compositebuffer_forcelink(); - zen::compress_forcelink(); - zen::filesystem_forcelink(); - zen::intmath_forcelink(); - zen::iobuffer_forcelink(); - zen::memory_forcelink(); - zen::mpscqueue_forcelink(); - zen::refcount_forcelink(); - zen::sha1_forcelink(); - zen::stats_forcelink(); - zen::stream_forcelink(); - zen::string_forcelink(); - zen::thread_forcelink(); - zen::timer_forcelink(); - zen::uid_forcelink(); - zen::uson_forcelink(); - zen::usonbuilder_forcelink(); - zen::usonpackage_forcelink(); - zen::crypto_forcelink(); -} -} // namespace zen - -# include <zencore/testing.h> - -namespace zen { - -TEST_CASE("Assert.Default") -{ - bool A = true; - bool B = false; - CHECK_THROWS_WITH(ZEN_ASSERT(A == B), "A == B"); -} - -TEST_CASE("Assert.Custom") -{ - struct MyAssertImpl : AssertImpl - { - ZEN_FORCENOINLINE ZEN_DEBUG_SECTION MyAssertImpl() : PrevAssertImpl(CurrentAssertImpl) { CurrentAssertImpl = this; } - virtual ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ~MyAssertImpl() { CurrentAssertImpl = PrevAssertImpl; } - virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION OnAssert(const char* Filename, - int LineNumber, - const char* FunctionName, - const char* Msg) - { - AssertFileName = Filename; - Line = LineNumber; - FuncName = FunctionName; - Message = Msg; - } - AssertImpl* PrevAssertImpl; - - const char* AssertFileName = nullptr; - int Line = -1; - const char* FuncName = nullptr; - const char* Message = nullptr; - }; - - MyAssertImpl MyAssert; - bool A = true; - bool B = false; - CHECK_THROWS_WITH(ZEN_ASSERT(A == B), "A == B"); - CHECK(MyAssert.AssertFileName != nullptr); - CHECK(MyAssert.Line != -1); - CHECK(MyAssert.FuncName != nullptr); - CHECK(strcmp(MyAssert.Message, "A == B") == 0); -} - -#endif - -} // namespace zen |