aboutsummaryrefslogtreecommitdiff
path: root/zencore
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2021-09-27 12:34:52 +0200
committerStefan Boberg <[email protected]>2021-09-27 12:34:52 +0200
commitf0036eada7f6bcf6e08afe3ea8517367ed73450e (patch)
treeb1ce3466bba36175cad369028fad1b410a34b5ec /zencore
parentFixed httpsys Windows compilation error (diff)
parentGetWindowsErrorAsString() -> GetSystemErrorAsString() (diff)
downloadzen-f0036eada7f6bcf6e08afe3ea8517367ed73450e.tar.xz
zen-f0036eada7f6bcf6e08afe3ea8517367ed73450e.zip
Merged latest from main
Diffstat (limited to 'zencore')
-rw-r--r--zencore/base64.cpp105
-rw-r--r--zencore/compactbinary.cpp710
-rw-r--r--zencore/except.cpp4
-rw-r--r--zencore/filesystem.cpp83
-rw-r--r--zencore/include/zencore/base64.h17
-rw-r--r--zencore/include/zencore/compactbinary.h135
-rw-r--r--zencore/include/zencore/compactbinaryvalue.h290
-rw-r--r--zencore/include/zencore/except.h2
-rw-r--r--zencore/include/zencore/filesystem.h5
-rw-r--r--zencore/include/zencore/iobuffer.h3
-rw-r--r--zencore/include/zencore/logging.h56
-rw-r--r--zencore/include/zencore/memory.h6
-rw-r--r--zencore/include/zencore/stats.h60
-rw-r--r--zencore/include/zencore/string.h319
-rw-r--r--zencore/include/zencore/zencore.h2
-rw-r--r--zencore/iobuffer.cpp26
-rw-r--r--zencore/stats.cpp241
-rw-r--r--zencore/thread.cpp6
-rw-r--r--zencore/timer.cpp14
-rw-r--r--zencore/zencore.vcxproj2
-rw-r--r--zencore/zencore.vcxproj.filters2
21 files changed, 1844 insertions, 244 deletions
diff --git a/zencore/base64.cpp b/zencore/base64.cpp
new file mode 100644
index 000000000..9a6ea3b75
--- /dev/null
+++ b/zencore/base64.cpp
@@ -0,0 +1,105 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/base64.h>
+
+namespace zen {
+
+/** The table used to encode a 6 bit value as an ascii character */
+static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
+ 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
+ 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+ 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};
+
+/** The table used to convert an ascii character into a 6 bit value */
+static const uint8_t DecodingAlphabet[256] = {
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x3E, 0xFF, 0xFF, 0xFF, 0x3F, // 0x20-0x2f
+ 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x30-0x3f
+ 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, // 0x40-0x4f
+ 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x50-0x5f
+ 0xFF, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, // 0x60-0x6f
+ 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x70-0x7f
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x80-0x8f
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x90-0x9f
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xa0-0xaf
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xb0-0xbf
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xc0-0xcf
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xd0-0xdf
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff
+};
+
+template<typename CharType>
+uint32_t
+Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest)
+{
+ CharType* EncodedBytes = Dest;
+
+ // Loop through the buffer converting 3 bytes of binary data at a time
+ while (Length >= 3)
+ {
+ uint8_t A = *Source++;
+ uint8_t B = *Source++;
+ uint8_t C = *Source++;
+ Length -= 3;
+
+ // The algorithm takes 24 bits of data (3 bytes) and breaks it into 4 6bit chunks represented as ascii
+ uint32_t ByteTriplet = A << 16 | B << 8 | C;
+
+ // Use the 6bit block to find the representation ascii character for it
+ EncodedBytes[3] = EncodingAlphabet[ByteTriplet & 0x3F];
+ ByteTriplet >>= 6;
+ EncodedBytes[2] = EncodingAlphabet[ByteTriplet & 0x3F];
+ ByteTriplet >>= 6;
+ EncodedBytes[1] = EncodingAlphabet[ByteTriplet & 0x3F];
+ ByteTriplet >>= 6;
+ EncodedBytes[0] = EncodingAlphabet[ByteTriplet & 0x3F];
+
+ // Now we can append this buffer to our destination string
+ EncodedBytes += 4;
+ }
+
+ // Since this algorithm operates on blocks, we may need to pad the last chunks
+ if (Length > 0)
+ {
+ uint8_t A = *Source++;
+ uint8_t B = 0;
+ uint8_t C = 0;
+ // Grab the second character if it is a 2 uint8_t finish
+ if (Length == 2)
+ {
+ B = *Source;
+ }
+ uint32_t ByteTriplet = A << 16 | B << 8 | C;
+ // Pad with = to make a 4 uint8_t chunk
+ EncodedBytes[3] = '=';
+ ByteTriplet >>= 6;
+ // If there's only one 1 uint8_t left in the source, then you need 2 pad chars
+ if (Length == 1)
+ {
+ EncodedBytes[2] = '=';
+ }
+ else
+ {
+ EncodedBytes[2] = EncodingAlphabet[ByteTriplet & 0x3F];
+ }
+ // Now encode the remaining bits the same way
+ ByteTriplet >>= 6;
+ EncodedBytes[1] = EncodingAlphabet[ByteTriplet & 0x3F];
+ ByteTriplet >>= 6;
+ EncodedBytes[0] = EncodingAlphabet[ByteTriplet & 0x3F];
+
+ EncodedBytes += 4;
+ }
+
+ // Add a null terminator
+ *EncodedBytes = 0;
+
+ return uint32_t(EncodedBytes - Dest);
+}
+
+template ZENCORE_API uint32_t Base64::Encode<char>(const uint8_t* Source, uint32_t Length, char* Dest);
+template ZENCORE_API uint32_t Base64::Encode<wchar_t>(const uint8_t* Source, uint32_t Length, wchar_t* Dest);
+
+} // namespace zen
diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp
index f4908aa9a..3b6d33e41 100644
--- a/zencore/compactbinary.cpp
+++ b/zencore/compactbinary.cpp
@@ -2,18 +2,35 @@
#include "zencore/compactbinary.h"
+#include <zencore/base64.h>
#include <zencore/compactbinaryvalidation.h>
+#include <zencore/compactbinaryvalue.h>
#include <zencore/compress.h>
#include <zencore/endian.h>
+#include <zencore/fmtutils.h>
#include <zencore/stream.h>
+#include <zencore/string.h>
#include <zencore/testing.h>
+#include <zencore/uid.h>
+#include <fmt/format.h>
#include <string_view>
+#if ZEN_WITH_TESTS
+# include <json11.hpp>
+# include <zencore/compactbinarybuilder.h>
+#endif
+
namespace zen {
const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365};
+double
+GetJulianDay(uint64_t Ticks)
+{
+ return (double)(1721425.5 + Ticks / TimeSpan::TicksPerDay);
+}
+
bool
IsLeapYear(int Year)
{
@@ -63,24 +80,254 @@ TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano)
Ticks = TotalTicks;
}
-//////////////////////////////////////////////////////////////////////////
+std::string
+TimeSpan::ToString(const char* Format) const
+{
+ using namespace fmt::literals;
-namespace CompactBinaryPrivate {
+ StringBuilder<128> Result;
- static constexpr const uint8_t GEmptyObjectPayload[] = {uint8_t(CbFieldType::Object), 0x00};
- static constexpr const uint8_t GEmptyArrayPayload[] = {uint8_t(CbFieldType::Array), 0x01, 0x00};
+ Result.Append((Ticks < 0) ? '-' : '+');
- template<typename T>
- static constexpr inline T ReadUnaligned(const void* const Memory)
+ while (*Format != '\0')
{
-#if PLATFORM_SUPPORTS_UNALIGNED_LOADS
- return *static_cast<const T*>(Memory);
-#else
- T Value;
- memcpy(&Value, Memory, sizeof(Value));
- return Value;
-#endif
+ if ((*Format == '%') && (*++Format != '\0'))
+ {
+ switch (*Format)
+ {
+ case 'd':
+ Result.Append("%i"_format(GetDays()));
+ break;
+ case 'D':
+ Result.Append("%08i"_format(GetDays()));
+ break;
+ case 'h':
+ Result.Append("%02i"_format(GetHours()));
+ break;
+ case 'm':
+ Result.Append("%02i"_format(GetMinutes()));
+ break;
+ case 's':
+ Result.Append("%02i"_format(GetSeconds()));
+ break;
+ case 'f':
+ Result.Append("%03i"_format(GetFractionMilli()));
+ break;
+ case 'u':
+ Result.Append("%06i"_format(GetFractionMicro()));
+ break;
+ case 't':
+ Result.Append("%07i"_format(GetFractionTicks()));
+ break;
+ case 'n':
+ Result.Append("%09i"_format(GetFractionNano()));
+ break;
+ default:
+ Result.Append(*Format);
+ }
+ }
+ else
+ {
+ Result.Append(*Format);
+ }
+
+ ++Format;
+ }
+
+ return Result.ToString();
+}
+
+std::string
+TimeSpan::ToString() const
+{
+ if (GetDays() == 0)
+ {
+ return ToString("%h:%m:%s.%f");
+ }
+
+ return ToString("%d.%h:%m:%s.%f");
+}
+
+int
+DateTime::GetYear() const
+{
+ int Year, Month, Day;
+ GetDate(Year, Month, Day);
+
+ return Year;
+}
+
+int
+DateTime::GetMonth() const
+{
+ int Year, Month, Day;
+ GetDate(Year, Month, Day);
+
+ return Month;
+}
+
+int
+DateTime::GetDay() const
+{
+ int Year, Month, Day;
+ GetDate(Year, Month, Day);
+
+ return Day;
+}
+
+int
+DateTime::GetHour() const
+{
+ return (int)((Ticks / TimeSpan::TicksPerHour) % 24);
+}
+
+int
+DateTime::GetHour12() const
+{
+ int Hour = GetHour();
+
+ if (Hour < 1)
+ {
+ return 12;
+ }
+
+ if (Hour > 12)
+ {
+ return (Hour - 12);
+ }
+
+ return Hour;
+}
+
+int
+DateTime::GetMinute() const
+{
+ return (int)((Ticks / TimeSpan::TicksPerMinute) % 60);
+}
+
+int
+DateTime::GetSecond() const
+{
+ return (int)((Ticks / TimeSpan::TicksPerSecond) % 60);
+}
+
+int
+DateTime::GetMillisecond() const
+{
+ return (int)((Ticks / TimeSpan::TicksPerMillisecond) % 1000);
+}
+
+void
+DateTime::GetDate(int& Year, int& Month, int& Day) const
+{
+ // Based on FORTRAN code in:
+ // Fliegel, H. F. and van Flandern, T. C.,
+ // Communications of the ACM, Vol. 11, No. 10 (October 1968).
+
+ int i, j, k, l, n;
+
+ l = int(GetJulianDay(Ticks) + 0.5) + 68569;
+ n = 4 * l / 146097;
+ l = l - (146097 * n + 3) / 4;
+ i = 4000 * (l + 1) / 1461001;
+ l = l - 1461 * i / 4 + 31;
+ j = 80 * l / 2447;
+ k = l - 2447 * j / 80;
+ l = j / 11;
+ j = j + 2 - 12 * l;
+ i = 100 * (n - 49) + i + l;
+
+ Year = i;
+ Month = j;
+ Day = k;
+}
+
+std::string
+DateTime::ToString(const char* Format) const
+{
+ using namespace fmt::literals;
+
+ StringBuilder<32> Result;
+ int Year, Month, Day;
+
+ GetDate(Year, Month, Day);
+
+ if (Format != nullptr)
+ {
+ while (*Format != '\0')
+ {
+ if ((*Format == '%') && (*(++Format) != '\0'))
+ {
+ switch (*Format)
+ {
+ // case 'a': Result.Append(IsMorning() ? TEXT("am") : TEXT("pm")); break;
+ // case 'A': Result.Append(IsMorning() ? TEXT("AM") : TEXT("PM")); break;
+ case 'd':
+ Result.Append("%02i"_format(Day));
+ break;
+ // case 'D': Result.Appendf(TEXT("%03i"), GetDayOfYear()); break;
+ case 'm':
+ Result.Append("%02i"_format(Month));
+ break;
+ case 'y':
+ Result.Append("%02i"_format(Year % 100));
+ break;
+ case 'Y':
+ Result.Append("%04i"_format(Year % 100));
+ break;
+ case 'h':
+ Result.Append("%02i"_format(GetHour12()));
+ break;
+ case 'H':
+ Result.Append("%02i"_format(GetHour()));
+ break;
+ case 'M':
+ Result.Append("%02i"_format(GetMinute()));
+ break;
+ case 'S':
+ Result.Append("%02i"_format(GetSecond()));
+ break;
+ case 's':
+ Result.Append("%03i"_format(GetMillisecond()));
+ break;
+ default:
+ Result.Append(*Format);
+ }
+ }
+ else
+ {
+ Result.Append(*Format);
+ }
+
+ // move to the next one
+ Format++;
+ }
}
+
+ return Result.ToString();
+}
+
+std::string
+DateTime::ToIso8601() const
+{
+ return ToString("%Y-%m-%dT%H:%M:%S.%sZ");
+}
+
+StringBuilderBase&
+Guid::ToString(StringBuilderBase& Sb) const
+{
+ char Buf[128];
+ snprintf(Buf, sizeof Buf, "%08x-%04x-%04x-%04x-%04x%08x", A, B >> 16, B & 0xFFFF, C >> 16, C & 0xFFFF, D);
+ Sb << Buf;
+
+ return Sb;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace CompactBinaryPrivate {
+ static constexpr const uint8_t GEmptyObjectPayload[] = {uint8_t(CbFieldType::Object), 0x00};
+ static constexpr const uint8_t GEmptyArrayPayload[] = {uint8_t(CbFieldType::Array), 0x01, 0x00};
} // namespace CompactBinaryPrivate
//////////////////////////////////////////////////////////////////////////
@@ -151,14 +398,10 @@ CbFieldView::AsArrayView()
MemoryView
CbFieldView::AsBinaryView(const MemoryView Default)
{
- if (CbFieldTypeOps::IsBinary(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsBinary(Accessor.GetType()))
{
- const uint8_t* const PayloadBytes = static_cast<const uint8_t*>(Payload);
- uint32_t ValueSizeByteCount;
- const uint64_t ValueSize = ReadVarUInt(PayloadBytes, ValueSizeByteCount);
-
Error = CbFieldError::None;
- return MemoryView(PayloadBytes + ValueSizeByteCount, ValueSize);
+ return Accessor.AsBinary();
}
else
{
@@ -170,20 +413,25 @@ CbFieldView::AsBinaryView(const MemoryView Default)
std::string_view
CbFieldView::AsString(const std::string_view Default)
{
- if (CbFieldTypeOps::IsString(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsString(Accessor.GetType()))
{
- const char* const PayloadChars = static_cast<const char*>(Payload);
- uint32_t ValueSizeByteCount;
- const uint64_t ValueSize = ReadVarUInt(PayloadChars, ValueSizeByteCount);
-
- if (ValueSize >= (uint64_t(1) << 31))
- {
- Error = CbFieldError::RangeError;
- return Default;
- }
+ Error = CbFieldError::None;
+ return Accessor.AsString();
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+std::u8string_view
+CbFieldView::AsU8String(const std::u8string_view Default)
+{
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsString(Accessor.GetType()))
+ {
Error = CbFieldError::None;
- return std::string_view(PayloadChars + ValueSizeByteCount, ValueSize);
+ return Accessor.AsU8String();
}
else
{
@@ -193,23 +441,11 @@ CbFieldView::AsString(const std::string_view Default)
}
uint64_t
-CbFieldView::AsInteger(const uint64_t Default, const IntegerParams Params)
+CbFieldView::AsInteger(const uint64_t Default, const CompactBinaryPrivate::IntegerParams Params)
{
- if (CbFieldTypeOps::IsInteger(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsInteger(Accessor.GetType()))
{
- // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero.
- const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1);
- const uint64_t IsNegative = uint8_t(Type) & 1;
-
- uint32_t MagnitudeByteCount;
- const uint64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount);
- const uint64_t Value = Magnitude ^ -int64_t(IsNegative);
-
- const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned);
- Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
-
- const uint64_t UseValueMask = -int64_t(IsInRange);
- return (Value & UseValueMask) | (Default & ~UseValueMask);
+ return Accessor.AsInteger(Params, &Error, Default);
}
else
{
@@ -221,25 +457,24 @@ CbFieldView::AsInteger(const uint64_t Default, const IntegerParams Params)
float
CbFieldView::AsFloat(const float Default)
{
- switch (CbFieldTypeOps::GetType(Type))
+ switch (CbValue Accessor = GetValue(); Accessor.GetType())
{
case CbFieldType::IntegerPositive:
case CbFieldType::IntegerNegative:
{
- const uint64_t IsNegative = uint8_t(Type) & 1;
+ const uint64_t IsNegative = uint8_t(Accessor.GetType()) & 1;
constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*FLT_MANT_DIG*/ 24) - 1);
uint32_t MagnitudeByteCount;
- const int64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount) + IsNegative;
+ const int64_t Magnitude = ReadVarUInt(Accessor.GetData(), MagnitudeByteCount) + IsNegative;
const uint64_t IsInRange = !(Magnitude & OutOfRangeMask);
Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
return IsInRange ? float(IsNegative ? -Magnitude : Magnitude) : Default;
}
case CbFieldType::Float32:
{
- Error = CbFieldError::None;
- const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint32_t>(Payload));
- return reinterpret_cast<const float&>(Value);
+ Error = CbFieldError::None;
+ return Accessor.AsFloat32();
}
case CbFieldType::Float64:
Error = CbFieldError::RangeError;
@@ -253,31 +488,29 @@ CbFieldView::AsFloat(const float Default)
double
CbFieldView::AsDouble(const double Default)
{
- switch (CbFieldTypeOps::GetType(Type))
+ switch (CbValue Accessor = GetValue(); Accessor.GetType())
{
case CbFieldType::IntegerPositive:
case CbFieldType::IntegerNegative:
{
- const uint64_t IsNegative = uint8_t(Type) & 1;
+ const uint64_t IsNegative = uint8_t(Accessor.GetType()) & 1;
constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*DBL_MANT_DIG*/ 53) - 1);
uint32_t MagnitudeByteCount;
- const int64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount) + IsNegative;
+ const int64_t Magnitude = ReadVarUInt(Accessor.GetData(), MagnitudeByteCount) + IsNegative;
const uint64_t IsInRange = !(Magnitude & OutOfRangeMask);
Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
return IsInRange ? double(IsNegative ? -Magnitude : Magnitude) : Default;
}
case CbFieldType::Float32:
{
- Error = CbFieldError::None;
- const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint32_t>(Payload));
- return reinterpret_cast<const float&>(Value);
+ Error = CbFieldError::None;
+ return Accessor.AsFloat32();
}
case CbFieldType::Float64:
{
- Error = CbFieldError::None;
- const uint64_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint64_t>(Payload));
- return reinterpret_cast<const double&>(Value);
+ Error = CbFieldError::None;
+ return Accessor.AsFloat64();
}
default:
Error = CbFieldError::TypeError;
@@ -288,19 +521,19 @@ CbFieldView::AsDouble(const double Default)
bool
CbFieldView::AsBool(const bool bDefault)
{
- const CbFieldType LocalType = Type;
- const bool bIsBool = CbFieldTypeOps::IsBool(LocalType);
- Error = bIsBool ? CbFieldError::None : CbFieldError::TypeError;
- return (uint8_t(bIsBool) & uint8_t(LocalType) & 1) | ((!bIsBool) & bDefault);
+ CbValue Accessor = GetValue();
+ const bool IsBool = CbFieldTypeOps::IsBool(Accessor.GetType());
+ Error = IsBool ? CbFieldError::None : CbFieldError::TypeError;
+ return (uint8_t(IsBool) & Accessor.AsBool()) | ((!IsBool) & bDefault);
}
IoHash
CbFieldView::AsObjectAttachment(const IoHash& Default)
{
- if (CbFieldTypeOps::IsObjectAttachment(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsObjectAttachment(Accessor.GetType()))
{
Error = CbFieldError::None;
- return IoHash::MakeFrom(Payload);
+ return Accessor.AsObjectAttachment();
}
else
{
@@ -312,10 +545,10 @@ CbFieldView::AsObjectAttachment(const IoHash& Default)
IoHash
CbFieldView::AsBinaryAttachment(const IoHash& Default)
{
- if (CbFieldTypeOps::IsBinaryAttachment(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsBinaryAttachment(Accessor.GetType()))
{
Error = CbFieldError::None;
- return IoHash::MakeFrom(Payload);
+ return Accessor.AsBinaryAttachment();
}
else
{
@@ -327,10 +560,10 @@ CbFieldView::AsBinaryAttachment(const IoHash& Default)
IoHash
CbFieldView::AsAttachment(const IoHash& Default)
{
- if (CbFieldTypeOps::IsAttachment(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsAttachment(Accessor.GetType()))
{
Error = CbFieldError::None;
- return IoHash::MakeFrom(Payload);
+ return Accessor.AsAttachment();
}
else
{
@@ -342,10 +575,10 @@ CbFieldView::AsAttachment(const IoHash& Default)
IoHash
CbFieldView::AsHash(const IoHash& Default)
{
- if (CbFieldTypeOps::IsHash(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsHash(Accessor.GetType()))
{
Error = CbFieldError::None;
- return IoHash::MakeFrom(Payload);
+ return Accessor.AsHash();
}
else
{
@@ -363,16 +596,10 @@ CbFieldView::AsUuid()
Guid
CbFieldView::AsUuid(const Guid& Default)
{
- if (CbFieldTypeOps::IsUuid(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsUuid(Accessor.GetType()))
{
Error = CbFieldError::None;
- Guid Value;
- memcpy(&Value, Payload, sizeof(Guid));
- Value.A = FromNetworkOrder(Value.A);
- Value.B = FromNetworkOrder(Value.B);
- Value.C = FromNetworkOrder(Value.C);
- Value.D = FromNetworkOrder(Value.D);
- return Value;
+ return Accessor.AsUuid();
}
else
{
@@ -390,12 +617,40 @@ CbFieldView::AsObjectId()
Oid
CbFieldView::AsObjectId(const Oid& Default)
{
- if (CbFieldTypeOps::IsObjectId(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsObjectId(Accessor.GetType()))
+ {
+ Error = CbFieldError::None;
+ return Accessor.AsObjectId();
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+CbCustomById
+CbFieldView::AsCustomById(CbCustomById Default)
+{
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsCustomById(Accessor.GetType()))
+ {
+ Error = CbFieldError::None;
+ return Accessor.AsCustomById();
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+CbCustomByName
+CbFieldView::AsCustomByName(CbCustomByName Default)
+{
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsCustomByName(Accessor.GetType()))
{
Error = CbFieldError::None;
- Oid Value;
- memcpy(&Value, Payload, sizeof(Oid));
- return Value;
+ return Accessor.AsCustomByName();
}
else
{
@@ -407,10 +662,10 @@ CbFieldView::AsObjectId(const Oid& Default)
int64_t
CbFieldView::AsDateTimeTicks(const int64_t Default)
{
- if (CbFieldTypeOps::IsDateTime(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsDateTime(Accessor.GetType()))
{
Error = CbFieldError::None;
- return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Payload));
+ return Accessor.AsDateTimeTicks();
}
else
{
@@ -434,10 +689,10 @@ CbFieldView::AsDateTime(DateTime Default)
int64_t
CbFieldView::AsTimeSpanTicks(const int64_t Default)
{
- if (CbFieldTypeOps::IsTimeSpan(Type))
+ if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsTimeSpan(Accessor.GetType()))
{
Error = CbFieldError::None;
- return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Payload));
+ return Accessor.AsTimeSpanTicks();
}
else
{
@@ -1146,6 +1401,259 @@ SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object)
//////////////////////////////////////////////////////////////////////////
+class CbJsonWriter
+{
+public:
+ explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; }
+
+ void WriteField(CbFieldView Field)
+ {
+ using namespace fmt::literals;
+ using namespace std::literals;
+
+ WriteOptionalComma();
+ WriteOptionalNewLine();
+
+ if (std::u8string_view Name = Field.GetU8Name(); !Name.empty())
+ {
+ AppendQuotedString(Name);
+ Builder << ": "sv;
+ }
+
+ switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+ {
+ case CbFieldType::Null:
+ Builder << "null"sv;
+ break;
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ {
+ Builder << '{';
+ NewLineAndIndent << '\t';
+ NeedsNewLine = true;
+ for (CbFieldView It : Field)
+ {
+ WriteField(It);
+ }
+ NewLineAndIndent.RemoveSuffix(1);
+ if (NeedsComma)
+ {
+ WriteOptionalNewLine();
+ }
+ Builder << '}';
+ }
+ break;
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ {
+ Builder << '[';
+ NewLineAndIndent << '\t';
+ NeedsNewLine = true;
+ for (CbFieldView It : Field)
+ {
+ WriteField(It);
+ }
+ NewLineAndIndent.RemoveSuffix(1);
+ if (NeedsComma)
+ {
+ WriteOptionalNewLine();
+ }
+ Builder << ']';
+ }
+ break;
+ case CbFieldType::Binary:
+ AppendBase64String(Accessor.AsBinary());
+ break;
+ case CbFieldType::String:
+ AppendQuotedString(Accessor.AsU8String());
+ break;
+ case CbFieldType::IntegerPositive:
+ Builder << Accessor.AsIntegerPositive();
+ break;
+ case CbFieldType::IntegerNegative:
+ Builder << Accessor.AsIntegerNegative();
+ break;
+ case CbFieldType::Float32:
+ Builder.Append("%.9g"_format(Accessor.AsFloat32()));
+ break;
+ case CbFieldType::Float64:
+ Builder.Append("%.17g"_format(Accessor.AsFloat64()));
+ break;
+ case CbFieldType::BoolFalse:
+ Builder << "false"sv;
+ break;
+ case CbFieldType::BoolTrue:
+ Builder << "true"sv;
+ break;
+ case CbFieldType::ObjectAttachment:
+ case CbFieldType::BinaryAttachment:
+ {
+ Builder << '"';
+ Accessor.AsAttachment().ToHexString(Builder);
+ Builder << '"';
+ }
+ break;
+ case CbFieldType::Hash:
+ {
+ Builder << '"';
+ Accessor.AsHash().ToHexString(Builder);
+ Builder << '"';
+ }
+ break;
+ case CbFieldType::Uuid:
+ {
+ Builder << '"';
+ Accessor.AsUuid().ToString(Builder);
+ Builder << '"';
+ }
+ break;
+ case CbFieldType::DateTime:
+ Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"';
+ break;
+ case CbFieldType::TimeSpan:
+ {
+ const TimeSpan Span(Accessor.AsTimeSpanTicks());
+ if (Span.GetDays() == 0)
+ {
+ Builder << '"' << Span.ToString("%h:%m:%s.%n") << '"';
+ }
+ else
+ {
+ Builder << '"' << Span.ToString("%d.%h:%m:%s.%n") << '"';
+ }
+ break;
+ }
+ case CbFieldType::ObjectId:
+ Builder << '"';
+ Accessor.AsObjectId().ToString(Builder);
+ Builder << '"';
+ break;
+ case CbFieldType::CustomById:
+ {
+ CbCustomById Custom = Accessor.AsCustomById();
+ Builder << "{ \"Id\": ";
+ Builder << Custom.Id;
+ Builder << ", \"Data\": ";
+ AppendBase64String(Custom.Data);
+ Builder << " }";
+ break;
+ }
+ case CbFieldType::CustomByName:
+ {
+ CbCustomByName Custom = Accessor.AsCustomByName();
+ Builder << "{ \"Name\": ";
+ AppendQuotedString(Custom.Name);
+ Builder << ", \"Data\": ";
+ AppendBase64String(Custom.Data);
+ Builder << " }";
+ break;
+ }
+ default:
+ ZEN_ASSERT(false);
+ break;
+ }
+
+ NeedsComma = true;
+ NeedsNewLine = true;
+ }
+
+private:
+ void WriteOptionalComma()
+ {
+ if (NeedsComma)
+ {
+ NeedsComma = false;
+ Builder << ',';
+ }
+ }
+
+ void WriteOptionalNewLine()
+ {
+ if (NeedsNewLine)
+ {
+ NeedsNewLine = false;
+ Builder << NewLineAndIndent;
+ }
+ }
+
+ void AppendQuotedString(std::u8string_view Value)
+ {
+ using namespace std::literals;
+
+ const AsciiSet EscapeSet(
+ "\\\"\b\f\n\r\t"
+ "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+ "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f");
+
+ Builder << '\"';
+ while (!Value.empty())
+ {
+ std::u8string_view Verbatim = AsciiSet::FindPrefixWithout(Value, EscapeSet);
+ Builder << Verbatim;
+
+ Value = Value.substr(Verbatim.size());
+
+ std::u8string_view Escape = AsciiSet::FindPrefixWith(Value, EscapeSet);
+ for (char Char : Escape)
+ {
+ switch (Char)
+ {
+ case '\\':
+ Builder << "\\\\"sv;
+ break;
+ case '\"':
+ Builder << "\\\""sv;
+ break;
+ case '\b':
+ Builder << "\\b"sv;
+ break;
+ case '\f':
+ Builder << "\\f"sv;
+ break;
+ case '\n':
+ Builder << "\\n"sv;
+ break;
+ case '\r':
+ Builder << "\\r"sv;
+ break;
+ case '\t':
+ Builder << "\\t"sv;
+ break;
+ default:
+ Builder << Char;
+ break;
+ }
+ }
+ Value = Value.substr(Escape.size());
+ }
+ Builder << '\"';
+ }
+
+ void AppendBase64String(MemoryView Value)
+ {
+ Builder << '"';
+ ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
+ const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+ const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
+ Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+ }
+
+private:
+ StringBuilderBase& Builder;
+ StringBuilder<32> NewLineAndIndent;
+ bool NeedsComma{false};
+ bool NeedsNewLine{false};
+};
+
+void
+CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder)
+{
+ CbJsonWriter Writer(Builder);
+ Writer.WriteField(Object.AsFieldView());
+}
+
+//////////////////////////////////////////////////////////////////////////
+
#if ZEN_WITH_TESTS
void
uson_forcelink()
@@ -1298,6 +1806,32 @@ TEST_CASE("uson.null")
CHECK(Field.IsNull() == false);
}
}
+
+TEST_CASE("uson.json")
+{
+ SUBCASE("string")
+ {
+ CbObjectWriter Writer;
+ Writer << "KeyOne"
+ << "ValueOne";
+ Writer << "KeyTwo"
+ << "ValueTwo";
+ CbObject Obj = Writer.Save();
+
+ StringBuilder<128> Sb;
+ const std::string_view JsonText = Obj.ToJson(Sb).ToView();
+
+ std::string JsonError;
+ json11::Json Json = json11::Json::parse(JsonText.data(), JsonError);
+
+ const std::string ValueOne = Json["KeyOne"].string_value();
+ const std::string ValueTwo = Json["KeyTwo"].string_value();
+
+ CHECK(JsonError.empty());
+ CHECK(ValueOne == "ValueOne");
+ CHECK(ValueTwo == "ValueTwo");
+ }
+}
#endif
} // namespace zen
diff --git a/zencore/except.cpp b/zencore/except.cpp
index 0167c406f..ebaecf815 100644
--- a/zencore/except.cpp
+++ b/zencore/except.cpp
@@ -72,11 +72,11 @@ ThrowSystemError(uint32_t ErrorCode, std::string_view Message)
std::string
GetLastErrorAsString()
{
- return GetErrorAsString(zen::GetLastError());
+ return GetSystemErrorAsString(zen::GetLastError());
}
std::string
-GetErrorAsString(uint32_t ErrorCode)
+GetSystemErrorAsString(uint32_t ErrorCode)
{
return std::error_code(ErrorCode, std::system_category()).message();
}
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index 8ddcbac52..d1b8b7aeb 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -7,6 +7,8 @@
#include <zencore/iobuffer.h>
#include <zencore/logging.h>
#include <zencore/string.h>
+#include <zencore/testing.h>
+
#if ZEN_PLATFORM_WINDOWS
# include <zencore/windows.h>
#endif
@@ -24,8 +26,6 @@
#endif
#include <filesystem>
-
-#include <doctest/doctest.h>
#include <gsl/gsl-lite.hpp>
namespace zen {
@@ -158,13 +158,11 @@ CleanDirectory(const wchar_t* DirPath)
{
return WipeDirectory(DirPath);
}
- else
- {
- return CreateDirectories(DirPath);
- }
+
+ return CreateDirectories(DirPath);
}
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
bool
CreateDirectories(const std::filesystem::path& Dir)
@@ -240,7 +238,7 @@ SupportsBlockRefCounting(std::filesystem::path Path)
return true;
#else
return false;
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
}
bool
@@ -404,7 +402,7 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
#else
ZEN_ERROR("CloneFile() is not implemented on this platform");
return false;
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
}
bool
@@ -445,7 +443,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
#else
ZEN_ERROR("CopyFile() is not implemented on this platform");
return false;
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
}
void
@@ -504,7 +502,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
{
ThrowLastError("File write failed for '{}'"_format(Path));
}
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
WriteSize -= ChunkSize;
DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize;
@@ -528,7 +526,7 @@ FileContents
ReadFile(std::filesystem::path Path)
{
uint64_t FileSizeBytes;
- void* Handle;
+ void* Handle;
#if ZEN_PLATFORM_WINDOWS
ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
@@ -545,7 +543,7 @@ ReadFile(std::filesystem::path Path)
}
FileSizeBytes = FileSize.EndOfFile.QuadPart;
- Handle = FromFile.Detach();
+ Handle = FromFile.Detach();
#else
int Fd = open(Path.c_str(), O_RDONLY);
if (Fd < 0)
@@ -558,7 +556,7 @@ ReadFile(std::filesystem::path Path)
fstat(Fd, &Stat);
FileSizeBytes = Stat.st_size;
- Handle = (void*)uintptr_t(Fd);
+ Handle = (void*)uintptr_t(Fd);
#endif
FileContents Contents;
@@ -599,7 +597,7 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi
#else
ZEN_ERROR("ScanFile() is not implemented on this platform");
return false;
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
}
std::string
@@ -622,11 +620,8 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
bool Continue = true;
CAtlFile RootDirHandle;
- HRESULT hRes = RootDirHandle.Create(RootDir.c_str(),
- GENERIC_READ,
- FILE_SHARE_READ | FILE_SHARE_WRITE,
- OPEN_EXISTING,
- FILE_FLAG_BACKUP_SEMANTICS);
+ HRESULT hRes =
+ RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS);
if (FAILED(hRes))
{
@@ -682,7 +677,9 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
}
else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE)
{
- ZEN_WARN("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), WideToUtf8(RootDir.c_str()));
+ ZEN_WARN("encountered device node during file system traversal: {} found in {}",
+ WideToUtf8(FileName),
+ WideToUtf8(RootDir.c_str()));
}
else
{
@@ -714,7 +711,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
{
const char* FileName = Entry->d_name;
- struct stat Stat;
+ struct stat Stat;
std::filesystem::path FullPath = RootDir / FileName;
stat(FullPath.c_str(), &Stat);
@@ -736,16 +733,16 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
else
{
ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}",
- Stat.st_mode, FileName, RootDir.c_str());
+ Stat.st_mode,
+ FileName,
+ RootDir.c_str());
}
}
closedir(Dir);
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
}
-
-
std::filesystem::path
PathFromHandle(void* NativeHandle)
{
@@ -776,7 +773,7 @@ PathFromHandle(void* NativeHandle)
return Buffer;
#else
# error Unimplemented platform
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
}
std::filesystem::path
@@ -798,16 +795,16 @@ GetRunningExecutablePath()
return Buffer;
#else
# error Unimplemented platform
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
}
-
-
//////////////////////////////////////////////////////////////////////////
//
// Testing related code follows...
//
+#if ZEN_WITH_TESTS
+
void
filesystem_forcelink()
{
@@ -824,24 +821,23 @@ TEST_CASE("filesystem")
// PathFromHandle
void* Handle;
-#if ZEN_PLATFORM_WINDOWS
- Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr,
- OPEN_EXISTING, 0, nullptr);
+# if ZEN_PLATFORM_WINDOWS
+ Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr);
CHECK(Handle != INVALID_HANDLE_VALUE);
-#else
+# else
int Fd = open(BinPath.c_str(), O_RDONLY);
CHECK(Fd >= 0);
Handle = (void*)uintptr_t(Fd);
-#endif
+# endif
auto FromHandle = PathFromHandle((void*)uintptr_t(Handle));
CHECK(equivalent(FromHandle, BinPath));
-#if ZEN_PLATFORM_WINDOWS
+# if ZEN_PLATFORM_WINDOWS
CloseHandle(Handle);
-#else
+# else
close(int(uintptr_t(Handle)));
-#endif
+# endif
// Traversal
struct : public FileSystemTraversal::TreeVisitor
@@ -851,13 +847,10 @@ TEST_CASE("filesystem")
bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
}
- virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override
- {
- return true;
- }
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
- bool bFoundExpected = false;
- std::filesystem::path Expected;
+ bool bFoundExpected = false;
+ std::filesystem::path Expected;
} Visitor;
Visitor.Expected = BinPath;
@@ -865,4 +858,6 @@ TEST_CASE("filesystem")
CHECK(Visitor.bFoundExpected);
}
+#endif
+
} // namespace zen
diff --git a/zencore/include/zencore/base64.h b/zencore/include/zencore/base64.h
new file mode 100644
index 000000000..4d78b085f
--- /dev/null
+++ b/zencore/include/zencore/base64.h
@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+struct Base64
+{
+ template<typename CharType>
+ static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest);
+
+ static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; }
+};
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
index 4fce129ea..ab01402f8 100644
--- a/zencore/include/zencore/compactbinary.h
+++ b/zencore/include/zencore/compactbinary.h
@@ -30,6 +30,7 @@ class CbArrayView;
class BinaryReader;
class BinaryWriter;
class CompressedBuffer;
+class CbValue;
class DateTime
{
@@ -41,8 +42,22 @@ public:
}
inline uint64_t GetTicks() const { return Ticks; }
- inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
- inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+ int GetYear() const;
+ int GetMonth() const;
+ int GetDay() const;
+ int GetHour() const;
+ int GetHour12() const;
+ int GetMinute() const;
+ int GetSecond() const;
+ int GetMillisecond() const;
+ void GetDate(int& Year, int& Month, int& Day) const;
+
+ inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
+ inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+ std::string ToString(const char* Format) const;
+ std::string ToIso8601() const;
private:
void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond);
@@ -98,6 +113,25 @@ public:
/** The number of timespan ticks per year (365 days, not accounting for leap years). */
static constexpr int64_t TicksPerYear = 365 * TicksPerDay;
+ int GetFractionTicks() const { return (int)(Ticks % TicksPerSecond); }
+
+ int GetFractionMicro() const { return (int)((Ticks % TicksPerSecond) / TicksPerMicrosecond); }
+
+ int GetFractionMilli() const { return (int)((Ticks % TicksPerSecond) / TicksPerMillisecond); }
+
+ int GetFractionNano() const { return (int)((Ticks % TicksPerSecond) * NanosecondsPerTick); }
+
+ int GetDays() const { return (int)(Ticks / TicksPerDay); }
+
+ int GetHours() const { return (int)((Ticks / TicksPerHour) % 24); }
+
+ int GetMinutes() const { return (int)((Ticks / TicksPerMinute) % 60); }
+
+ int GetSeconds() const { return (int)((Ticks / TicksPerSecond) % 60); }
+
+ ZENCORE_API std::string ToString(const char* Format) const;
+ ZENCORE_API std::string ToString() const;
+
private:
void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano);
@@ -107,6 +141,8 @@ private:
struct Guid
{
uint32_t A, B, C, D;
+
+ StringBuilderBase& ToString(StringBuilderBase& OutString) const;
};
//////////////////////////////////////////////////////////////////////////
@@ -318,6 +354,9 @@ public:
static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; }
static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; }
+ static constexpr inline bool IsCustomById(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomById; }
+ static constexpr inline bool IsCustomByName(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomByName; }
+
static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; }
static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; }
@@ -365,6 +404,46 @@ public:
virtual void VisitTimeSpan(TimeSpan Value) = 0;
};
+/** A custom compact binary field type with an integer identifier. */
+struct CbCustomById
+{
+ /** An identifier for the sub-type of the field. */
+ uint64_t Id = 0;
+ /** A view of the value. Lifetime is tied to the field that the value is associated with. */
+ MemoryView Data;
+};
+
+/** A custom compact binary field type with a string identifier. */
+struct CbCustomByName
+{
+ /** An identifier for the sub-type of the field. Lifetime is tied to the field that the name is associated with. */
+ std::u8string_view Name;
+ /** A view of the value. Lifetime is tied to the field that the value is associated with. */
+ MemoryView Data;
+};
+
+namespace CompactBinaryPrivate {
+ /** Parameters for converting to an integer. */
+ struct IntegerParams
+ {
+ /** Whether the output type has a sign bit. */
+ uint32_t IsSigned : 1;
+ /** Bits of magnitude. (7 for int8) */
+ uint32_t MagnitudeBits : 31;
+ };
+
+ /** Make integer params for the given integer type. */
+ template<typename IntType>
+ static constexpr inline IntegerParams MakeIntegerParams()
+ {
+ IntegerParams Params;
+ Params.IsSigned = IntType(-1) < IntType(0);
+ Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned;
+ return Params;
+ }
+
+} // namespace CompactBinaryPrivate
+
/**
* An atom of data in the compact binary format.
*
@@ -393,13 +472,25 @@ public:
ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType);
+ /** Construct a field from a value, without access to the name. */
+ inline explicit CbFieldView(const CbValue& Value);
+
/** Returns the name of the field if it has a name, otherwise an empty view. */
constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); }
+ /** Returns the name of the field if it has a name, otherwise an empty view. */
+ constexpr inline std::u8string_view GetU8Name() const
+ {
+ return std::u8string_view(static_cast<const char8_t*>(Payload) - NameLen, NameLen);
+ }
+
+ /** Returns the value for unchecked access. Prefer the typed accessors below. */
+ inline CbValue GetValue() const;
ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView());
ZENCORE_API CbObjectView AsObjectView();
ZENCORE_API CbArrayView AsArrayView();
ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view());
+ ZENCORE_API std::u8string_view AsU8String(std::u8string_view Default = std::u8string_view());
ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const;
@@ -448,6 +539,11 @@ public:
/** Access the field as a OID. Returns the provided default on error. */
ZENCORE_API Oid AsObjectId(const Oid& Default);
+ /** Access the field as a custom sub-type with an integer identifier. Returns the provided default on error. */
+ ZENCORE_API CbCustomById AsCustomById(CbCustomById Default);
+ /** Access the field as a custom sub-type with a string identifier. Returns the provided default on error. */
+ ZENCORE_API CbCustomByName AsCustomByName(CbCustomByName Default);
+
/** Access the field as a date/time tick count. Returns the provided default on error. */
ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0);
@@ -590,25 +686,6 @@ protected:
}
private:
- /** Parameters for converting to an integer. */
- struct IntegerParams
- {
- /** Whether the output type has a sign bit. */
- uint32_t IsSigned : 1;
- /** Bits of magnitude. (7 for int8) */
- uint32_t MagnitudeBits : 31;
- };
-
- /** Make integer params for the given integer type. */
- template<typename IntType>
- static constexpr inline IntegerParams MakeIntegerParams()
- {
- IntegerParams Params;
- Params.IsSigned = IntType(-1) < IntType(0);
- Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned;
- return Params;
- }
-
/**
* Access the field as the given integer type.
*
@@ -617,11 +694,12 @@ private:
template<typename IntType>
inline IntType AsInteger(IntType Default)
{
- return IntType(AsInteger(uint64_t(Default), MakeIntegerParams<IntType>()));
+ return IntType(AsInteger(uint64_t(Default), CompactBinaryPrivate::MakeIntegerParams<IntType>()));
}
- ZENCORE_API uint64_t AsInteger(uint64_t Default, IntegerParams Params);
+ ZENCORE_API uint64_t AsInteger(uint64_t Default, CompactBinaryPrivate::IntegerParams Params);
+private:
/** The field type, with the transient HasFieldType flag if the field contains its type. */
CbFieldType Type = CbFieldType::None;
/** The error (if any) that occurred on the last field access. */
@@ -869,6 +947,11 @@ private:
inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {}
};
+/**
+ * Serialize a compact binary object to JSON.
+ */
+ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder);
+
class CbObjectView : protected CbFieldView
{
friend class CbFieldView;
@@ -951,6 +1034,12 @@ public:
/** Whether the field has a value. */
using CbFieldView::operator bool;
+ StringBuilderBase& ToJson(StringBuilderBase& Builder) const
+ {
+ CompactBinaryToJson(*this, Builder);
+ return Builder;
+ }
+
private:
friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); }
friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); }
diff --git a/zencore/include/zencore/compactbinaryvalue.h b/zencore/include/zencore/compactbinaryvalue.h
new file mode 100644
index 000000000..5795ef957
--- /dev/null
+++ b/zencore/include/zencore/compactbinaryvalue.h
@@ -0,0 +1,290 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+#include <zencore/endian.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/memory.h>
+
+namespace zen {
+
+namespace CompactBinaryPrivate {
+
+ template<typename T>
+ static constexpr inline T ReadUnaligned(const void* const Memory)
+ {
+#if PLATFORM_SUPPORTS_UNALIGNED_LOADS
+ return *static_cast<const T*>(Memory);
+#else
+ T Value;
+ memcpy(&Value, Memory, sizeof(Value));
+ return Value;
+#endif
+ }
+} // namespace CompactBinaryPrivate
+/**
+ * A type that provides unchecked access to compact binary values.
+ *
+ * The main purpose of the type is to efficiently switch on field type. For every other use case,
+ * prefer to use the field, array, and object types directly. The accessors here do not check the
+ * type before reading the value, which means they can read out of bounds even on a valid compact
+ * binary value if the wrong accessor is used.
+ */
+class CbValue
+{
+public:
+ CbValue(CbFieldType Type, const void* Value);
+
+ CbObjectView AsObjectView() const;
+ CbArrayView AsArrayView() const;
+
+ MemoryView AsBinary() const;
+
+ /** Access as a string. Checks for range errors and uses the default if OutError is not null. */
+ std::string_view AsString(CbFieldError* OutError = nullptr, std::string_view Default = std::string_view()) const;
+
+ /** Access as a string as UTF8. Checks for range errors and uses the default if OutError is not null. */
+ std::u8string_view AsU8String(CbFieldError* OutError = nullptr, std::u8string_view Default = std::u8string_view()) const;
+
+ /**
+ * Access as an integer, with both positive and negative values returned as unsigned.
+ *
+ * Checks for range errors and uses the default if OutError is not null.
+ */
+ uint64_t AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError = nullptr, uint64_t Default = 0) const;
+
+ uint64_t AsIntegerPositive() const;
+ int64_t AsIntegerNegative() const;
+
+ float AsFloat32() const;
+ double AsFloat64() const;
+
+ bool AsBool() const;
+
+ inline IoHash AsObjectAttachment() const { return AsHash(); }
+ inline IoHash AsBinaryAttachment() const { return AsHash(); }
+ inline IoHash AsAttachment() const { return AsHash(); }
+
+ IoHash AsHash() const;
+ Guid AsUuid() const;
+
+ int64_t AsDateTimeTicks() const;
+ int64_t AsTimeSpanTicks() const;
+
+ Oid AsObjectId() const;
+
+ CbCustomById AsCustomById() const;
+ CbCustomByName AsCustomByName() const;
+
+ inline CbFieldType GetType() const { return Type; }
+ inline const void* GetData() const { return Data; }
+
+private:
+ const void* Data;
+ CbFieldType Type;
+};
+
+inline CbFieldView::CbFieldView(const CbValue& InValue) : Type(InValue.GetType()), Payload(InValue.GetData())
+{
+}
+
+inline CbValue
+CbFieldView::GetValue() const
+{
+ return CbValue(CbFieldTypeOps::GetType(Type), Payload);
+}
+
+inline CbValue::CbValue(CbFieldType InType, const void* InValue) : Data(InValue), Type(InType)
+{
+}
+
+inline CbObjectView
+CbValue::AsObjectView() const
+{
+ return CbObjectView(*this);
+}
+
+inline CbArrayView
+CbValue::AsArrayView() const
+{
+ return CbArrayView(*this);
+}
+
+inline MemoryView
+CbValue::AsBinary() const
+{
+ const uint8_t* const Bytes = static_cast<const uint8_t*>(Data);
+ uint32_t ValueSizeByteCount;
+ const uint64_t ValueSize = ReadVarUInt(Bytes, ValueSizeByteCount);
+ return MakeMemoryView(Bytes + ValueSizeByteCount, ValueSize);
+}
+
+inline std::string_view
+CbValue::AsString(CbFieldError* OutError, std::string_view Default) const
+{
+ const char* const Chars = static_cast<const char*>(Data);
+ uint32_t ValueSizeByteCount;
+ const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
+
+ if (OutError)
+ {
+ if (ValueSize >= (uint64_t(1) << 31))
+ {
+ *OutError = CbFieldError::RangeError;
+ return Default;
+ }
+ *OutError = CbFieldError::None;
+ }
+
+ return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+}
+
+inline std::u8string_view
+CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const
+{
+ const char8_t* const Chars = static_cast<const char8_t*>(Data);
+ uint32_t ValueSizeByteCount;
+ const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
+
+ if (OutError)
+ {
+ if (ValueSize >= (uint64_t(1) << 31))
+ {
+ *OutError = CbFieldError::RangeError;
+ return Default;
+ }
+ *OutError = CbFieldError::None;
+ }
+
+ return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+}
+
+inline uint64_t
+CbValue::AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError, uint64_t Default) const
+{
+ // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero.
+ const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1);
+ const uint64_t IsNegative = uint8_t(Type) & 1;
+
+ uint32_t MagnitudeByteCount;
+ const uint64_t Magnitude = ReadVarUInt(Data, MagnitudeByteCount);
+ const uint64_t Value = Magnitude ^ -int64_t(IsNegative);
+
+ if (OutError)
+ {
+ const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned);
+ *OutError = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
+
+ const uint64_t UseValueMask = -int64_t(IsInRange);
+ return (Value & UseValueMask) | (Default & ~UseValueMask);
+ }
+
+ return Value;
+}
+
+inline uint64_t
+CbValue::AsIntegerPositive() const
+{
+ uint32_t MagnitudeByteCount;
+ return ReadVarUInt(Data, MagnitudeByteCount);
+}
+
+inline int64_t
+CbValue::AsIntegerNegative() const
+{
+ uint32_t MagnitudeByteCount;
+ return int64_t(ReadVarUInt(Data, MagnitudeByteCount)) ^ -int64_t(1);
+}
+
+inline float
+CbValue::AsFloat32() const
+{
+ const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint32_t>(Data));
+ return reinterpret_cast<const float&>(Value);
+}
+
+inline double
+CbValue::AsFloat64() const
+{
+ const uint64_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint64_t>(Data));
+ return reinterpret_cast<const double&>(Value);
+}
+
+inline bool
+CbValue::AsBool() const
+{
+ return uint8_t(Type) & 1;
+}
+
+inline IoHash
+CbValue::AsHash() const
+{
+ return IoHash::MakeFrom(Data);
+}
+
+inline Guid
+CbValue::AsUuid() const
+{
+ Guid Value;
+ memcpy(&Value, Data, sizeof(Guid));
+ Value.A = FromNetworkOrder(Value.A);
+ Value.B = FromNetworkOrder(Value.B);
+ Value.C = FromNetworkOrder(Value.C);
+ Value.D = FromNetworkOrder(Value.D);
+ return Value;
+}
+
+inline int64_t
+CbValue::AsDateTimeTicks() const
+{
+ return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data));
+}
+
+inline int64_t
+CbValue::AsTimeSpanTicks() const
+{
+ return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data));
+}
+
+inline Oid
+CbValue::AsObjectId() const
+{
+ return Oid::FromMemory(Data);
+}
+
+inline CbCustomById
+CbValue::AsCustomById() const
+{
+ const uint8_t* Bytes = static_cast<const uint8_t*>(Data);
+ uint32_t DataSizeByteCount;
+ const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount);
+ Bytes += DataSizeByteCount;
+
+ CbCustomById Value;
+ uint32_t TypeIdByteCount;
+ Value.Id = ReadVarUInt(Bytes, TypeIdByteCount);
+ Value.Data = MakeMemoryView(Bytes + TypeIdByteCount, DataSize - TypeIdByteCount);
+ return Value;
+}
+
+inline CbCustomByName
+CbValue::AsCustomByName() const
+{
+ const uint8_t* Bytes = static_cast<const uint8_t*>(Data);
+ uint32_t DataSizeByteCount;
+ const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount);
+ Bytes += DataSizeByteCount;
+
+ uint32_t TypeNameLenByteCount;
+ const uint64_t TypeNameLen = ReadVarUInt(Bytes, TypeNameLenByteCount);
+ Bytes += TypeNameLenByteCount;
+
+ CbCustomByName Value;
+ Value.Name = std::u8string_view(reinterpret_cast<const char8_t*>(Bytes), static_cast<std::u8string_view::size_type>(TypeNameLen));
+ Value.Data = MakeMemoryView(Bytes + TypeNameLen, DataSize - TypeNameLen - TypeNameLenByteCount);
+ return Value;
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h
index 1dc6209d6..e0e4aaae0 100644
--- a/zencore/include/zencore/except.h
+++ b/zencore/include/zencore/except.h
@@ -27,7 +27,7 @@ ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const st
ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message);
ZENCORE_API std::string GetLastErrorAsString();
-ZENCORE_API std::string GetErrorAsString(uint32_t ErrorCode);
+ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode);
inline int32_t
GetLastError()
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index 66ab37e5c..6678528f6 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -14,19 +14,16 @@ class IoBuffer;
/** Delete directory (after deleting any contents)
*/
-ZENCORE_API bool DeleteDirectories(const wchar_t* dir);
ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir);
/** Ensure directory exists.
Will also create any required parent directories
*/
-ZENCORE_API bool CreateDirectories(const wchar_t* dir);
ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir);
/** Ensure directory exists and delete contents (if any) before returning
*/
-ZENCORE_API bool CleanDirectory(const wchar_t* dir);
ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir);
/** Map native file handle to a path
@@ -82,6 +79,6 @@ public:
//////////////////////////////////////////////////////////////////////////
-void filesystem_forcelink(); // internal
+void filesystem_forcelink(); // internal
} // namespace zen
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 298952dd6..6f3609d19 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -11,6 +11,7 @@
namespace zen {
+struct IoHash;
struct IoBufferExtendedCore;
enum class ZenContentType : uint8_t
@@ -352,6 +353,8 @@ public:
inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
};
+IoHash HashBuffer(IoBuffer& Buffer);
+
void iobuffer_forcelink();
} // namespace zen
diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h
index 221f5f358..0838cfe80 100644
--- a/zencore/include/zencore/logging.h
+++ b/zencore/include/zencore/logging.h
@@ -39,46 +39,46 @@ using zen::Log;
// Helper macros for logging
-#define ZEN_TRACE(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().trace(fmtstr##sv, ##__VA_ARGS__);\
+#define ZEN_TRACE(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().trace(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
-#define ZEN_DEBUG(fmtstr, ...) \
+#define ZEN_DEBUG(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().debug(fmtstr##sv, ##__VA_ARGS__); \
+ } while (false)
+
+#define ZEN_INFO(fmtstr, ...) \
do \
{ \
using namespace std::literals; \
- Log().debug(fmtstr##sv, ##__VA_ARGS__);\
+ Log().info(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
-#define ZEN_INFO(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().info(fmtstr##sv, ##__VA_ARGS__);\
- } while (false)
-
-#define ZEN_WARN(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().warn(fmtstr##sv, ##__VA_ARGS__);\
- } while (false)
-
-#define ZEN_ERROR(fmtstr, ...) \
+#define ZEN_WARN(fmtstr, ...) \
do \
{ \
using namespace std::literals; \
- Log().error(fmtstr##sv, ##__VA_ARGS__);\
+ Log().warn(fmtstr##sv, ##__VA_ARGS__); \
+ } while (false)
+
+#define ZEN_ERROR(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().error(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
-#define ZEN_CRITICAL(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().critical(fmtstr##sv, ##__VA_ARGS__);\
+#define ZEN_CRITICAL(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().critical(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
#define ZEN_CONSOLE(fmtstr, ...) \
diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h
index 3d4db1081..aba391c85 100644
--- a/zencore/include/zencore/memory.h
+++ b/zencore/include/zencore/memory.h
@@ -354,6 +354,12 @@ MakeMemoryView(const void* Data, const void* DataEnd)
return MemoryView(Data, DataEnd);
}
+[[nodiscard]] inline MemoryView
+MakeMemoryView(const void* Data, uint64_t Size)
+{
+ return MemoryView(Data, reinterpret_cast<const uint8_t*>(Data) + Size);
+}
+
/**
* Make a non-owning mutable view of the memory of the initializer list.
*
diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h
index 7290fd914..0554f620d 100644
--- a/zencore/include/zencore/stats.h
+++ b/zencore/include/zencore/stats.h
@@ -12,30 +12,44 @@ template<typename T>
class Gauge
{
public:
- Gauge() : m_value{0} {}
+ Gauge() : m_Value{0} {}
+
+ T Value() const { return m_Value; }
+ void SetValue(T Value) { m_Value = Value; }
private:
- T m_value;
+ std::atomic<T> m_Value;
};
+/** Stats counter
+ *
+ * A counter is modified by adding or subtracting a value from a current value.
+ * This would typically be used to track number of requests in flight, number
+ * of active jobs etc
+ *
+ */
class Counter
{
public:
inline void SetValue(uint64_t Value) { m_count = Value; }
inline uint64_t Value() const { return m_count; }
- inline void Increment(int64_t AddValue) { m_count += AddValue; }
- inline void Decrement(int64_t SubValue) { m_count -= SubValue; }
- inline void Clear() { m_count = 0; }
+ inline void Increment(int64_t AddValue) { m_count.fetch_add(AddValue); }
+ inline void Decrement(int64_t SubValue) { m_count.fetch_sub(SubValue); }
+ inline void Clear() { m_count.store(0, std::memory_order_release); }
private:
- std::atomic_uint64_t m_count{0};
+ std::atomic<uint64_t> m_count{0};
};
-/// <summary>
-/// Exponential Weighted Moving Average
-/// </summary>
-class EWMA
+/** Exponential Weighted Moving Average
+
+ This is very raw, to use as little state as possible. If we
+ want to use this more broadly in user code we should perhaps
+ add a more user-friendly wrapper
+ */
+
+class RawEWMA
{
public:
/// <summary>
@@ -49,16 +63,38 @@ public:
double Rate() const;
private:
- double m_rate = 0;
+ std::atomic<double> m_Rate = 0;
};
/// <summary>
-/// Tracks rate of events over time (i.e requests/sec)
+/// Tracks rate of events over time (i.e requests/sec), using
+/// exponential moving averages
/// </summary>
class Meter
{
public:
+ Meter();
+ ~Meter();
+
+ double Rate1(); // One-minute rate
+ double Rate5(); // Five-minute rate
+ double Rate15(); // Fifteen-minute rate
+ double MeanRate(); // Mean rate since instantiation of this meter
+ void Mark(uint64_t Count = 1); // Register one or more events
+
private:
+ std::atomic<uint64_t> m_TotalCount{0}; // Accumulator counting number of marks since beginning
+ std::atomic<uint64_t> m_PendingCount{0}; // Pending EWMA update accumulator
+ std::atomic<uint64_t> m_StartTick{0}; // Time this was instantiated (for mean)
+ std::atomic<uint64_t> m_LastTick{0}; // Timestamp of last EWMA tick
+ std::atomic<int64_t> m_Remain{0}; // Tracks the "modulo" of tick time
+ bool m_IsFirstTick = true;
+ RawEWMA m_RateM1;
+ RawEWMA m_RateM5;
+ RawEWMA m_RateM15;
+
+ void TickIfNecessary();
+ void Tick();
};
extern void stats_forcelink();
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index bb9b1c896..2c0d10577 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -14,6 +14,8 @@
#include <span>
#include <string_view>
+#include <type_traits>
+
namespace zen {
//////////////////////////////////////////////////////////////////////////
@@ -94,6 +96,14 @@ public:
const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete;
const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete;
+ inline size_t AddUninitialized(size_t Count)
+ {
+ EnsureCapacity(Count);
+ const size_t OldCount = Size();
+ m_CurPos += Count;
+ return OldCount;
+ }
+
StringBuilderImpl& Append(C OneChar)
{
EnsureCapacity(1);
@@ -209,6 +219,12 @@ public:
return AppendRange(String.data(), String.data() + String.size());
}
+ inline void RemoveSuffix(int32_t Count)
+ {
+ ZEN_ASSERT(Count <= Size());
+ m_CurPos -= Count;
+ }
+
inline const C* c_str() const
{
EnsureNulTerminated();
@@ -322,6 +338,12 @@ protected:
extern template class StringBuilderImpl<char>;
+inline StringBuilderImpl<char>&
+operator<<(StringBuilderImpl<char>& Builder, char Char)
+{
+ return Builder.Append(Char);
+}
+
class StringBuilderBase : public StringBuilderImpl<char>
{
public:
@@ -661,6 +683,303 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
//////////////////////////////////////////////////////////////////////////
+/**
+ * ASCII character bitset useful for fast and readable parsing
+ *
+ * Entirely constexpr. Works with both wide and narrow strings.
+ *
+ * Example use cases:
+ *
+ * constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n");
+ * bool bIsWhitespace = WhitespaceCharacters.Test(MyChar);
+ * const char* HelloWorld = AsciiSet::Skip(" \t\tHello world!", WhitespaceCharacters);
+ *
+ * constexpr AsciiSet XmlEscapeChars("&<>\"'");
+ * check(AsciiSet::HasNone(EscapedXmlString, XmlEscapeChars));
+ *
+ * constexpr AsciiSet Delimiters(".:;");
+ * const TCHAR* DelimiterOrEnd = AsciiSet::FindFirstOrEnd(PrefixedName, Delimiters);
+ * FString Prefix(PrefixedName, DelimiterOrEnd - PrefixedName);
+ *
+ * constexpr AsciiSet Slashes("/\\");
+ * const TCHAR* SlashOrEnd = AsciiSet::FindLastOrEnd(PathName, Slashes);
+ * const TCHAR* FileName = *SlashOrEnd ? SlashOrEnd + 1 : PathName;
+ */
+class AsciiSet
+{
+public:
+ template<typename CharType, int N>
+ constexpr AsciiSet(const CharType (&Chars)[N]) : AsciiSet(StringToBitset(Chars))
+ {
+ }
+
+ /** Returns true if a character is part of the set */
+ template<typename CharType>
+ constexpr inline bool Contains(CharType Char) const
+ {
+ using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+ return !!TestImpl((UnsignedCharType)Char);
+ }
+
+ /** Returns non-zero if a character is part of the set. Prefer Contains() to avoid VS2019 conversion warnings. */
+ template<typename CharType>
+ constexpr inline uint64_t Test(CharType Char) const
+ {
+ using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+ return TestImpl((UnsignedCharType)Char);
+ }
+
+ /** Create new set with specified character in it */
+ constexpr inline AsciiSet operator+(char Char) const
+ {
+ using UnsignedCharType = std::make_unsigned<char>::type;
+
+ InitData Bitset = {LoMask, HiMask};
+ SetImpl(Bitset, (UnsignedCharType)Char);
+ return AsciiSet(Bitset);
+ }
+
+ /** Create new set containing inverse set of characters - likely including null-terminator */
+ constexpr inline AsciiSet operator~() const { return AsciiSet(~LoMask, ~HiMask); }
+
+ ////////// Algorithms for C strings //////////
+
+ /** Find first character of string inside set or end pointer. Never returns null. */
+ template<class CharType>
+ static constexpr const CharType* FindFirstOrEnd(const CharType* Str, AsciiSet Set)
+ {
+ for (AsciiSet SetOrNil(Set.LoMask | NilMask, Set.HiMask); !SetOrNil.Test(*Str); ++Str)
+ ;
+
+ return Str;
+ }
+
+ /** Find last character of string inside set or end pointer. Never returns null. */
+ template<class CharType>
+ static constexpr const CharType* FindLastOrEnd(const CharType* Str, AsciiSet Set)
+ {
+ const CharType* Last = FindFirstOrEnd(Str, Set);
+
+ for (const CharType* It = Last; *It; It = FindFirstOrEnd(It + 1, Set))
+ {
+ Last = It;
+ }
+
+ return Last;
+ }
+
+ /** Find first character of string outside of set. Never returns null. */
+ template<typename CharType>
+ static constexpr const CharType* Skip(const CharType* Str, AsciiSet Set)
+ {
+ while (Set.Contains(*Str))
+ {
+ ++Str;
+ }
+
+ return Str;
+ }
+
+ /** Test if string contains any character in set */
+ template<typename CharType>
+ static constexpr bool HasAny(const CharType* Str, AsciiSet Set)
+ {
+ return *FindFirstOrEnd(Str, Set) != '\0';
+ }
+
+ /** Test if string contains no character in set */
+ template<typename CharType>
+ static constexpr bool HasNone(const CharType* Str, AsciiSet Set)
+ {
+ return *FindFirstOrEnd(Str, Set) == '\0';
+ }
+
+ /** Test if string contains any character outside of set */
+ template<typename CharType>
+ static constexpr bool HasOnly(const CharType* Str, AsciiSet Set)
+ {
+ return *Skip(Str, Set) == '\0';
+ }
+
+ ////////// Algorithms for string types like FStringView and FString //////////
+
+ /** Get initial substring with all characters in set */
+ template<class StringType>
+ static constexpr StringType FindPrefixWith(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Forward, EInclude::Members, EKeep::Head>(Str, Set);
+ }
+
+ /** Get initial substring with no characters in set */
+ template<class StringType>
+ static constexpr StringType FindPrefixWithout(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Head>(Str, Set);
+ }
+
+ /** Trim initial characters in set */
+ template<class StringType>
+ static constexpr StringType TrimPrefixWith(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Forward, EInclude::Members, EKeep::Tail>(Str, Set);
+ }
+
+ /** Trim initial characters not in set */
+ template<class StringType>
+ static constexpr StringType TrimPrefixWithout(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Tail>(Str, Set);
+ }
+
+ /** Get trailing substring with all characters in set */
+ template<class StringType>
+ static constexpr StringType FindSuffixWith(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Reverse, EInclude::Members, EKeep::Tail>(Str, Set);
+ }
+
+ /** Get trailing substring with no characters in set */
+ template<class StringType>
+ static constexpr StringType FindSuffixWithout(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Tail>(Str, Set);
+ }
+
+ /** Trim trailing characters in set */
+ template<class StringType>
+ static constexpr StringType TrimSuffixWith(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Reverse, EInclude::Members, EKeep::Head>(Str, Set);
+ }
+
+ /** Trim trailing characters not in set */
+ template<class StringType>
+ static constexpr StringType TrimSuffixWithout(const StringType& Str, AsciiSet Set)
+ {
+ return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Head>(Str, Set);
+ }
+
+ /** Test if string contains any character in set */
+ template<class StringType>
+ static constexpr bool HasAny(const StringType& Str, AsciiSet Set)
+ {
+ return !HasNone(Str, Set);
+ }
+
+ /** Test if string contains no character in set */
+ template<class StringType>
+ static constexpr bool HasNone(const StringType& Str, AsciiSet Set)
+ {
+ uint64_t Match = 0;
+ for (auto Char : Str)
+ {
+ Match |= Set.Test(Char);
+ }
+ return Match == 0;
+ }
+
+ /** Test if string contains any character outside of set */
+ template<class StringType>
+ static constexpr bool HasOnly(const StringType& Str, AsciiSet Set)
+ {
+ auto End = Str.data() + Str.size();
+ return FindFirst<EInclude::Members>(Set, GetData(Str), End) == End;
+ }
+
+private:
+ enum class EDir
+ {
+ Forward,
+ Reverse
+ };
+ enum class EInclude
+ {
+ Members,
+ NonMembers
+ };
+ enum class EKeep
+ {
+ Head,
+ Tail
+ };
+
+ template<EInclude Include, typename CharType>
+ static constexpr const CharType* FindFirst(AsciiSet Set, const CharType* It, const CharType* End)
+ {
+ for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); ++It)
+ ;
+ return It;
+ }
+
+ template<EInclude Include, typename CharType>
+ static constexpr const CharType* FindLast(AsciiSet Set, const CharType* It, const CharType* End)
+ {
+ for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); --It)
+ ;
+ return It;
+ }
+
+ template<EDir Dir, EInclude Include, EKeep Keep, class StringType>
+ static constexpr StringType Scan(const StringType& Str, AsciiSet Set)
+ {
+ auto Begin = Str.data();
+ auto End = Begin + Str.size();
+ auto It = Dir == EDir::Forward ? FindFirst<Include>(Set, Begin, End) : FindLast<Include>(Set, End - 1, Begin - 1) + 1;
+
+ return Keep == EKeep::Head ? StringType(Begin, static_cast<int32_t>(It - Begin)) : StringType(It, static_cast<int32_t>(End - It));
+ }
+
+ // Work-around for constexpr limitations
+ struct InitData
+ {
+ uint64_t Lo, Hi;
+ };
+ static constexpr uint64_t NilMask = uint64_t(1) << '\0';
+
+ static constexpr inline void SetImpl(InitData& Bitset, uint32_t Char)
+ {
+ uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0);
+ uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1);
+ uint64_t Bit = uint64_t(1) << uint8_t(Char & 0x3f);
+
+ Bitset.Lo |= Bit & IsLo;
+ Bitset.Hi |= Bit & IsHi;
+ }
+
+ constexpr inline uint64_t TestImpl(uint32_t Char) const
+ {
+ uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0);
+ uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1);
+ uint64_t Bit = uint64_t(1) << (Char & 0x3f);
+
+ return (Bit & IsLo & LoMask) | (Bit & IsHi & HiMask);
+ }
+
+ template<typename CharType, int N>
+ static constexpr InitData StringToBitset(const CharType (&Chars)[N])
+ {
+ using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+ InitData Bitset = {0, 0};
+ for (int I = 0; I < N - 1; ++I)
+ {
+ SetImpl(Bitset, UnsignedCharType(Chars[I]));
+ }
+
+ return Bitset;
+ }
+
+ constexpr AsciiSet(InitData Bitset) : LoMask(Bitset.Lo), HiMask(Bitset.Hi) {}
+
+ constexpr AsciiSet(uint64_t Lo, uint64_t Hi) : LoMask(Lo), HiMask(Hi) {}
+
+ uint64_t LoMask, HiMask;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
void string_forcelink(); // internal
} // namespace zen
diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h
index f6093cb96..4b9c1af1b 100644
--- a/zencore/include/zencore/zencore.h
+++ b/zencore/include/zencore/zencore.h
@@ -102,9 +102,11 @@
// Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable.
# define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name))
# define ZEN_FORCENOINLINE __declspec(noinline) /* Force code to NOT be inline */
+# define LINE_TERMINATOR_ANSI "\r\n"
#else
# define ZEN_CODE_SECTION(Name)
# define ZEN_FORCENOINLINE
+# define LINE_TERMINATOR_ANSI "\n"
#endif
#if ZEN_ARCH_ARM64
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index 5d3458dba..a730a316f 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -5,6 +5,7 @@
#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
+#include <zencore/iohash.h>
#include <zencore/logging.h>
#include <zencore/memory.h>
#include <zencore/testing.h>
@@ -36,7 +37,7 @@ IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment)
return VirtualAlloc(nullptr, InSize, MEM_COMMIT, PAGE_READWRITE);
}
else
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
{
return Memory::Alloc(InSize, Alignment);
}
@@ -51,7 +52,7 @@ IoBufferCore::FreeBuffer()
VirtualFree(const_cast<void*>(m_DataPtr), 0, MEM_DECOMMIT);
}
else
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
{
return Memory::Free(const_cast<void*>(m_DataPtr));
}
@@ -183,7 +184,7 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
#if ZEN_PLATFORM_WINDOWS
BOOL Success = CloseHandle(m_FileHandle);
#else
- int Fd = int(uintptr_t(m_FileHandle));
+ int Fd = int(uintptr_t(m_FileHandle));
bool Success = (close(Fd) == 0);
#endif
@@ -241,17 +242,17 @@ IoBufferExtendedCore::Materialize() const
/* FileOffsetLow */ uint32_t(MapOffset & 0xffFFffFFu),
/* dwNumberOfBytesToMap */ MapSize);
#else
- m_MmapHandle = (void*)uintptr_t(~MapSize); // ~ so it's never null (assuming MapSize >= 0)
+ m_MmapHandle = (void*)uintptr_t(~MapSize); // ~ so it's never null (assuming MapSize >= 0)
m_Flags |= kOwnsMmap;
void* MappedBase = mmap(
/* addr */ nullptr,
/* length */ MapSize,
/* prot */ PROT_READ,
- /* flags */ MAP_SHARED|MAP_NORESERVE,
+ /* flags */ MAP_SHARED | MAP_NORESERVE,
/* fd */ int(uintptr_t(m_FileHandle)),
/* offset */ MapOffset);
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
if (MappedBase == nullptr)
{
@@ -378,7 +379,7 @@ IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint
struct stat Stat;
fstat(Fd, &Stat);
FileSize = Stat.st_size;
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
// TODO: should validate that offset is in range
@@ -415,7 +416,7 @@ IoBuffer
IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName)
{
uint64_t FileSize;
- void* Handle;
+ void* Handle;
#if ZEN_PLATFORM_WINDOWS
CAtlFile DataFile;
@@ -447,7 +448,7 @@ IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName)
FileSize = Stat.st_size;
Handle = (void*)uintptr_t(Fd);
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
IoBuffer Iob(IoBuffer::File, Handle, 0, FileSize);
Iob.m_Core->SetIsWholeFile(true);
@@ -455,6 +456,13 @@ IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName)
return Iob;
}
+IoHash
+HashBuffer(IoBuffer& Buffer)
+{
+ // TODO: handle disk buffers with special path
+ return IoHash::HashBuffer(Buffer.Data(), Buffer.Size());
+}
+
//////////////////////////////////////////////////////////////////////////
#if ZEN_WITH_TESTS
diff --git a/zencore/stats.cpp b/zencore/stats.cpp
index c5187940e..9ae2ddd28 100644
--- a/zencore/stats.cpp
+++ b/zencore/stats.cpp
@@ -2,6 +2,7 @@
#include "zencore/stats.h"
#include <cmath>
+#include "zencore/thread.h"
#include "zencore/timer.h"
#if ZEN_WITH_TESTS
@@ -14,62 +15,254 @@
namespace zen {
-static constexpr int kTickInterval = 5; // In seconds
-static constexpr double kSecondsPerMinute = 60.0;
-static constexpr int kOneMinute = 1;
-static constexpr int kFiveMinutes = 5;
-static constexpr int kFifteenMinutes = 15;
+static constinit int kTickIntervalInSeconds = 5;
+static constinit double kSecondsPerMinute = 60.0;
+static constinit int kOneMinute = 1;
+static constinit int kFiveMinutes = 5;
+static constinit int kFifteenMinutes = 15;
-static double kM1_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kOneMinute);
-static double kM5_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kFiveMinutes);
-static double kM15_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kFifteenMinutes);
+static const double kM1_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kOneMinute);
+static const double kM5_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kFiveMinutes);
+static const double kM15_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kFifteenMinutes);
-static uint64_t CountPerTick = GetHifreqTimerFrequencySafe() * kTickInterval;
-static uint64_t CountPerSecond = GetHifreqTimerFrequencySafe();
+static const uint64_t CountPerTick = GetHifreqTimerFrequencySafe() * kTickIntervalInSeconds;
+static const uint64_t CountPerSecond = GetHifreqTimerFrequencySafe();
+
+//////////////////////////////////////////////////////////////////////////
void
-EWMA::Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate)
+RawEWMA::Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate)
{
- double InstantRate = double(Count) / Interval;
+ const double InstantRate = double(Count) / Interval;
if (IsInitialUpdate)
{
- m_rate = InstantRate;
+ m_Rate.store(InstantRate, std::memory_order_release);
}
else
{
- m_rate += Alpha * (InstantRate - m_rate);
+ m_Rate.fetch_add(Alpha * (InstantRate - m_Rate));
}
}
double
-EWMA::Rate() const
+RawEWMA::Rate() const
{
- return m_rate * CountPerSecond;
+ return m_Rate.load(std::memory_order_relaxed) * CountPerSecond;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+Meter::Meter() : m_StartTick{GetHifreqTimerValue()}, m_LastTick(m_StartTick.load())
+{
+}
+
+Meter::~Meter()
+{
+}
+
+void
+Meter::TickIfNecessary()
+{
+ uint64_t OldTick = m_LastTick.load();
+ const uint64_t NewTick = GetHifreqTimerValue();
+ const uint64_t Age = NewTick - OldTick;
+
+ if (Age > CountPerTick)
+ {
+ // Ensure only one thread at a time updates the time. This
+ // works because our tick interval should be sufficiently
+ // long to ensure two threads don't end up inside this block
+
+ if (m_LastTick.compare_exchange_strong(OldTick, NewTick))
+ {
+ m_Remain.fetch_add(Age);
+
+ do
+ {
+ int64_t Remain = m_Remain.load(std::memory_order_relaxed);
+
+ if (Remain < 0)
+ {
+ return;
+ }
+
+ if (m_Remain.compare_exchange_strong(Remain, Remain - CountPerTick))
+ {
+ Tick();
+ }
+ } while (true);
+ }
+ }
+}
+
+void
+Meter::Tick()
+{
+ const uint64_t PendingCount = m_PendingCount.exchange(0);
+ const bool IsFirstTick = m_IsFirstTick;
+
+ if (IsFirstTick)
+ {
+ m_IsFirstTick = false;
+ }
+
+ m_RateM1.Tick(kM1_ALPHA, CountPerTick, PendingCount, IsFirstTick);
+ m_RateM5.Tick(kM5_ALPHA, CountPerTick, PendingCount, IsFirstTick);
+ m_RateM15.Tick(kM15_ALPHA, CountPerTick, PendingCount, IsFirstTick);
+}
+
+double
+Meter::Rate1()
+{
+ TickIfNecessary();
+
+ return m_RateM1.Rate();
+}
+
+double
+Meter::Rate5()
+{
+ TickIfNecessary();
+
+ return m_RateM5.Rate();
+}
+
+double
+Meter::Rate15()
+{
+ TickIfNecessary();
+
+ return m_RateM15.Rate();
+}
+
+double
+Meter::MeanRate()
+{
+ const uint64_t Count = m_TotalCount.load(std::memory_order_relaxed);
+
+ if (Count == 0)
+ {
+ return 0.0;
+ }
+
+ const uint64_t Elapsed = GetHifreqTimerValue() - m_StartTick;
+
+ return (double(Count) * GetHifreqTimerFrequency()) / Elapsed;
+}
+
+void
+Meter::Mark(uint64_t Count)
+{
+ TickIfNecessary();
+
+ m_TotalCount.fetch_add(Count);
+ m_PendingCount.fetch_add(Count);
}
//////////////////////////////////////////////////////////////////////////
#if ZEN_WITH_TESTS
-TEST_CASE("Stats")
+TEST_CASE("EWMA")
{
- SUBCASE("Simple")
+ SUBCASE("Simple_1")
+ {
+ RawEWMA Ewma1;
+ Ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true);
+
+ CHECK(fabs(Ewma1.Rate() - 5) < 0.1);
+
+ for (int i = 0; i < 60; ++i)
+ {
+ Ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false);
+ }
+
+ CHECK(fabs(Ewma1.Rate() - 10) < 0.1);
+
+ for (int i = 0; i < 60; ++i)
+ {
+ Ewma1.Tick(kM1_ALPHA, CountPerSecond, 20, false);
+ }
+
+ CHECK(fabs(Ewma1.Rate() - 20) < 0.1);
+ }
+
+ SUBCASE("Simple_10")
{
- EWMA ewma1;
- ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true);
+ RawEWMA Ewma1;
+ RawEWMA Ewma5;
+ RawEWMA Ewma15;
+ Ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true);
+ Ewma5.Tick(kM5_ALPHA, CountPerSecond, 5, true);
+ Ewma15.Tick(kM15_ALPHA, CountPerSecond, 5, true);
+
+ CHECK(fabs(Ewma1.Rate() - 5) < 0.1);
+ CHECK(fabs(Ewma5.Rate() - 5) < 0.1);
+ CHECK(fabs(Ewma15.Rate() - 5) < 0.1);
- CHECK(ewma1.Rate() - 5 < 0.001);
+ auto Tick1 = [&Ewma1](auto Value) { Ewma1.Tick(kM1_ALPHA, CountPerSecond, Value, false); };
+ auto Tick5 = [&Ewma5](auto Value) { Ewma5.Tick(kM5_ALPHA, CountPerSecond, Value, false); };
+ auto Tick15 = [&Ewma15](auto Value) { Ewma15.Tick(kM15_ALPHA, CountPerSecond, Value, false); };
for (int i = 0; i < 60; ++i)
- ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false);
+ {
+ Tick1(10);
+ Tick5(10);
+ Tick15(10);
+ }
+
+ CHECK(fabs(Ewma1.Rate() - 10) < 0.1);
+
+ for (int i = 0; i < 5 * 60; ++i)
+ {
+ Tick1(20);
+ Tick5(20);
+ Tick15(20);
+ }
- CHECK(ewma1.Rate() - 10 < 0.001);
+ CHECK(fabs(Ewma1.Rate() - 20) < 0.1);
+ CHECK(fabs(Ewma5.Rate() - 20) < 0.1);
- ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false);
+ for (int i = 0; i < 16 * 60; ++i)
+ {
+ Tick1(100);
+ Tick5(100);
+ Tick15(100);
+ }
+
+ CHECK(fabs(Ewma1.Rate() - 100) < 0.1);
+ CHECK(fabs(Ewma5.Rate() - 100) < 0.1);
+ CHECK(fabs(Ewma15.Rate() - 100) < 0.5);
}
}
+# if 0 // This is not really a unit test, but mildly useful to exercise some code
+TEST_CASE("Meter")
+{
+ Meter Meter1;
+ Meter1.Mark(1);
+ Sleep(1000);
+ Meter1.Mark(1);
+ Sleep(1000);
+ Meter1.Mark(1);
+ Sleep(1000);
+ Meter1.Mark(1);
+ Sleep(1000);
+ Meter1.Mark(1);
+ Sleep(1000);
+ Meter1.Mark(1);
+ Sleep(1000);
+ Meter1.Mark(1);
+ Sleep(1000);
+ Meter1.Mark(1);
+ Sleep(1000);
+ Meter1.Mark(1);
+ Sleep(1000);
+ [[maybe_unused]] double Rate = Meter1.MeanRate();
+}
+# endif
+
void
stats_forcelink()
{
diff --git a/zencore/thread.cpp b/zencore/thread.cpp
index 748a0b146..20ab19f56 100644
--- a/zencore/thread.cpp
+++ b/zencore/thread.cpp
@@ -157,7 +157,7 @@ NamedMutex::Exists(std::string_view MutexName)
return true;
}
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
#if ZEN_PLATFORM_WINDOWS
@@ -260,7 +260,7 @@ ProcessHandle::Wait(int TimeoutMs)
return false;
}
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
//////////////////////////////////////////////////////////////////////////
@@ -329,7 +329,7 @@ ProcessMonitor::IsActive() const
return m_ProcessHandles.empty() == false;
}
-#endif // ZEN_PLATFORM_WINDOWS
+#endif // ZEN_PLATFORM_WINDOWS
//////////////////////////////////////////////////////////////////////////
diff --git a/zencore/timer.cpp b/zencore/timer.cpp
index 1e73a7532..88ec89cb7 100644
--- a/zencore/timer.cpp
+++ b/zencore/timer.cpp
@@ -30,7 +30,7 @@ GetHifreqTimerValue()
}
uint64_t
-internalGetHifreqTimerFrequency()
+InternalGetHifreqTimerFrequency()
{
#if ZEN_PLATFORM_WINDOWS
LARGE_INTEGER li;
@@ -42,21 +42,23 @@ internalGetHifreqTimerFrequency()
#endif
}
-static uint64_t qpcFreq = internalGetHifreqTimerFrequency();
+static uint64_t QpcFreq = InternalGetHifreqTimerFrequency();
uint64_t
GetHifreqTimerFrequency()
{
- return qpcFreq;
+ return QpcFreq;
}
uint64_t
GetHifreqTimerFrequencySafe()
{
- if (!qpcFreq)
- qpcFreq = internalGetHifreqTimerFrequency();
+ if (!QpcFreq)
+ {
+ QpcFreq = InternalGetHifreqTimerFrequency();
+ }
- return qpcFreq;
+ return QpcFreq;
}
//////////////////////////////////////////////////////////////////////////
diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj
index 2322f7173..3adf779ed 100644
--- a/zencore/zencore.vcxproj
+++ b/zencore/zencore.vcxproj
@@ -113,6 +113,7 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="include\zencore\atomic.h" />
+ <ClInclude Include="include\zencore\base64.h" />
<ClInclude Include="include\zencore\blake3.h" />
<ClInclude Include="include\zencore\compositebuffer.h" />
<ClInclude Include="include\zencore\crc32.h" />
@@ -157,6 +158,7 @@
<ClInclude Include="include\zencore\zencore.h" />
</ItemGroup>
<ItemGroup>
+ <ClCompile Include="base64.cpp" />
<ClCompile Include="blake3.cpp" />
<ClCompile Include="compositebuffer.cpp" />
<ClCompile Include="compress.cpp" />
diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters
index d2e7a3159..92aa0db1d 100644
--- a/zencore/zencore.vcxproj.filters
+++ b/zencore/zencore.vcxproj.filters
@@ -44,6 +44,7 @@
<ClInclude Include="include\zencore\testutils.h" />
<ClInclude Include="include\zencore\testing.h" />
<ClInclude Include="include\zencore\mpscqueue.h" />
+ <ClInclude Include="include\zencore\base64.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="sha1.cpp" />
@@ -77,6 +78,7 @@
<ClCompile Include="session.cpp" />
<ClCompile Include="testutils.cpp" />
<ClCompile Include="mpscqueue.cpp" />
+ <ClCompile Include="base64.cpp" />
</ItemGroup>
<ItemGroup>
<Filter Include="CAS">