From a90c4fd0d6c4c18c24efc60fe26f2f043c7221be Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 23 Sep 2021 08:24:13 +0200 Subject: Use /check/health instead of /test/hello. --- zenserver/upstream/upstreamcache.cpp | 4 ++-- zenserver/upstream/zen.cpp | 4 ++-- zenserver/upstream/zen.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index a889fb984..9d43462c0 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -317,9 +317,9 @@ namespace detail { ZenStructuredCacheSession Session(*m_Client); ZenCacheResult Result; - for (int32_t Attempt = 0, MaxAttempts = 3; Attempt < MaxAttempts && !Result.Success; ++Attempt) + for (int32_t Attempt = 0, MaxAttempts = 2; Attempt < MaxAttempts && !Result.Success; ++Attempt) { - Result = Session.SayHello(); + Result = Session.CheckHealth(); } m_HealthOk = Result.ErrorCode == 0; diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 710d381c6..530bed32a 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -382,10 +382,10 @@ ZenStructuredCacheSession::~ZenStructuredCacheSession() } ZenCacheResult -ZenStructuredCacheSession::SayHello() +ZenStructuredCacheSession::CheckHealth() { ExtendableStringBuilder<256> Uri; - Uri << m_Client.ServiceUrl() << "/test/hello"; + Uri << m_Client.ServiceUrl() << "/health/check"; cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index 48886096d..158be668a 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -111,7 +111,7 @@ public: ZenStructuredCacheSession(ZenStructuredCacheClient& OuterClient); ~ZenStructuredCacheSession(); - ZenCacheResult SayHello(); + ZenCacheResult CheckHealth(); ZenCacheResult GetCacheRecord(std::string_view BucketId, const IoHash& Key, ZenContentType Type); ZenCacheResult GetCachePayload(std::string_view BucketId, const IoHash& Key, const IoHash& PayloadId); ZenCacheResult PutCacheRecord(std::string_view BucketId, const IoHash& Key, IoBuffer Value, ZenContentType Type); -- cgit v1.2.3 From 105826c3e349eff6ddcd45dec4af81f9a3279f42 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 23 Sep 2021 12:10:56 +0200 Subject: Fixed dumb bug in CasChunkSet::IterateChunks which would cause infinite loop --- zenstore/CAS.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index eaf72cb41..1db2b50bf 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -50,7 +50,7 @@ CasChunkSet::RemoveChunksIf(std::function&& P void CasChunkSet::IterateChunks(std::function&& Callback) { - for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd;) + for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd; ++It) { Callback(*It); } -- cgit v1.2.3 From c20289a3bd781682b5ba6afebdaf228fb202f42c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 23 Sep 2021 12:38:13 +0200 Subject: cidstore: made all updates log using LogMapping() also fixed issue with FindChunkByCid() which would assert when it should not --- zenstore/cidstore.cpp | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index 08a3192ff..d76058bd1 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -45,11 +45,23 @@ struct CidStore::Impl ZEN_ASSERT(Compressed != IoHash::Zero); RwLock::ExclusiveLockScope _(m_Lock); - m_CidMap.insert_or_assign(DecompressedId, Compressed); - // TODO: it's pretty wasteful to log even idempotent updates - // however we can't simply use the boolean returned by insert_or_assign - // since there's not a 1:1 mapping between compressed and uncompressed - // so if we want a last-write-wins policy then we have to log each update + + auto It = m_CidMap.try_emplace(DecompressedId, Compressed); + if (!It.second) + { + if (It.first.value() != Compressed) + { + It.first.value() = Compressed; + } + else + { + // No point logging an update that won't change anything + return; + } + } + + // It's not ideal to do this while holding the lock in case + // we end up in blocking I/O but that's for later LogMapping(DecompressedId, Compressed); } @@ -68,6 +80,10 @@ struct CidStore::Impl { CompressedHash = It->second; } + else + { + return {}; + } } ZEN_ASSERT(CompressedHash != IoHash::Zero); @@ -84,7 +100,7 @@ struct CidStore::Impl if (It == m_CidMap.end()) { - // Not in map, or tombstone + // Not in map return false; } @@ -171,7 +187,7 @@ struct CidStore::Impl const IoHash& BadHash = It->first; // Log a tombstone record - m_LogFile.Append({.Uncompressed = BadHash, .Compressed = IoHash::Zero}); + LogMapping(BadHash, IoHash::Zero); BadChunks.push_back(BadHash); -- cgit v1.2.3 From 4842881e1b2924a1d5dcac7b7e6da6dcc7558456 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 23 Sep 2021 13:40:10 +0200 Subject: Added HashBuffer(IoBuffer&) overload with trivial (but inappropriate for the longer term) implementation --- zencore/include/zencore/iobuffer.h | 3 +++ zencore/iobuffer.cpp | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 517cc7b69..6ee40e468 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -9,6 +9,7 @@ namespace zen { +struct IoHash; struct IoBufferExtendedCore; enum class ZenContentType : uint8_t @@ -348,6 +349,8 @@ public: inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); } }; +IoHash HashBuffer(IoBuffer& Buffer); + void iobuffer_forcelink(); } // namespace zen diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index bcecc768f..dc998d5ea 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -381,6 +382,13 @@ IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName) return {}; } +IoHash +HashBuffer(IoBuffer& Buffer) +{ + // TODO: handle disk buffers with special path + return IoHash::HashBuffer(Buffer.Data(), Buffer.Size()); +} + ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS -- cgit v1.2.3 From af88b1a47dfeae66a0b2faa1623ff6c59d435e22 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 23 Sep 2021 13:48:34 +0200 Subject: Added scrubbing logic to ZenCacheDiskLayer/ZenCacheMemoryLayer This currently only goes through the motions of hashing the data to verify it, but does not perform recovery nor does it validate referential integrity --- zenserver/cache/structuredcachestore.cpp | 201 +++++++++++++++++++++++-------- zenserver/cache/structuredcachestore.h | 1 + 2 files changed, 153 insertions(+), 49 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 3d80bb14c..4f16b81f2 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -131,23 +131,18 @@ ZenCacheMemoryLayer::~ZenCacheMemoryLayer() bool ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue) { - CacheBucket* Bucket = nullptr; - - { - RwLock::SharedLockScope _(m_Lock); + RwLock::SharedLockScope _(m_Lock); - auto it = m_Buckets.find(std::string(InBucket)); + auto it = m_Buckets.find(std::string(InBucket)); - if (it != m_Buckets.end()) - { - Bucket = &it->second; - } + if (it == m_Buckets.end()) + { + return false; } - if (Bucket == nullptr) - return false; + CacheBucket* Bucket = Bucket = &it->second; - ZEN_ASSERT(Bucket != nullptr); + _.ReleaseNow(); return Bucket->Get(HashKey, OutValue); } @@ -177,8 +172,6 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Bucket = &m_Buckets[std::string(InBucket)]; } - ZEN_ASSERT(Bucket != nullptr); - // Note that since the underlying IoBuffer is retained, the content type is also Bucket->Put(HashKey, Value); @@ -195,7 +188,31 @@ ZenCacheMemoryLayer::DropBucket(std::string_view Bucket) void ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx) { - ZEN_UNUSED(Ctx); + RwLock::SharedLockScope _(m_Lock); + + for (auto& Kv : m_Buckets) + { + Kv.second.Scrub(Ctx); + } +} + +void +ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx) +{ + std::vector BadHashes; + + for (auto& Kv : m_cacheMap) + { + if (Kv.first != IoHash::HashBuffer(Kv.second)) + { + BadHashes.push_back(Kv.first); + } + } + + if (!BadHashes.empty()) + { + Ctx.ReportBadChunks(BadHashes); + } } bool @@ -203,16 +220,16 @@ ZenCacheMemoryLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutV { RwLock::SharedLockScope _(m_bucketLock); - auto bucketIt = m_cacheMap.find(HashKey); - - if (bucketIt == m_cacheMap.end()) + if (auto bucketIt = m_cacheMap.find(HashKey); bucketIt == m_cacheMap.end()) { return false; } + else + { + OutValue.Value = bucketIt->second; - OutValue.Value = bucketIt->second; - - return true; + return true; + } } void @@ -241,8 +258,19 @@ struct DiskLocation static uint64_t CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags) { return Offset | Flags; } - inline uint64_t Offset() const { return OffsetAndFlags & kOffsetMask; } - inline uint64_t IsFlagSet(uint64_t Flag) const { return OffsetAndFlags & Flag; } + inline uint64_t Offset() const { return OffsetAndFlags & kOffsetMask; } + inline uint64_t IsFlagSet(uint64_t Flag) const { return OffsetAndFlags & Flag; } + inline ZenContentType GetContentType() const + { + ZenContentType ContentType = ZenContentType::kBinary; + + if (IsFlagSet(DiskLocation::kStructured)) + { + ContentType = ZenContentType::kCbObject; + } + + return ContentType; + } }; struct DiskIndexEntry @@ -264,9 +292,14 @@ struct ZenCacheDiskLayer::CacheBucket static bool Delete(std::filesystem::path BucketDir); bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); + + bool GetStandaloneCacheValue(const IoHash& HashKey, ZenCacheValue& OutValue, const DiskLocation& Loc); + + bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); void Put(const IoHash& HashKey, const ZenCacheValue& Value); void Drop(); void Flush(); + void Scrub(ScrubContext& Ctx); inline bool IsOk() const { return m_Ok; } @@ -406,6 +439,37 @@ ZenCacheDiskLayer::CacheBucket::BuildPath(WideStringBuilderBase& Path, const IoH Path.AppendAsciiRange(HexString + 5, HexString + sizeof(HexString)); } +bool +ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue) +{ + if (!Loc.IsFlagSet(DiskLocation::kStandaloneFile)) + { + OutValue.Value = IoBufferBuilder::MakeFromFileHandle(m_SobsFile.Handle(), Loc.Offset(), Loc.Size); + OutValue.Value.SetContentType(Loc.GetContentType()); + + return true; + } + + return false; +} + +bool +ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const IoHash& HashKey, ZenCacheValue& OutValue, const DiskLocation& Loc) +{ + WideStringBuilder<128> DataFilePath; + BuildPath(DataFilePath, HashKey); + + if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str())) + { + OutValue.Value = Data; + OutValue.Value.SetContentType(Loc.GetContentType()); + + return true; + } + + return false; +} + bool ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue) { @@ -420,35 +484,14 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal { const DiskLocation& Loc = it->second; - ZenContentType ContentType = ZenContentType::kBinary; - - if (Loc.IsFlagSet(DiskLocation::kStructured)) - { - ContentType = ZenContentType::kCbObject; - } - - if (!Loc.IsFlagSet(DiskLocation::kStandaloneFile)) + if (GetInlineCacheValue(Loc, OutValue)) { - OutValue.Value = IoBufferBuilder::MakeFromFileHandle(m_SobsFile.Handle(), Loc.Offset(), Loc.Size); - OutValue.Value.SetContentType(ContentType); - return true; } - else - { - _.ReleaseNow(); - WideStringBuilder<128> DataFilePath; - BuildPath(DataFilePath, HashKey); + _.ReleaseNow(); - if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str())) - { - OutValue.Value = Data; - OutValue.Value.SetContentType(ContentType); - - return true; - } - } + return GetStandaloneCacheValue(HashKey, OutValue, Loc); } return false; @@ -518,9 +561,58 @@ ZenCacheDiskLayer::CacheBucket::Flush() } void -ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) +ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx) { - ZEN_UNUSED(Ctx); + std::vector StandaloneFiles; + + std::vector BadChunks; + std::vector BadStandaloneChunks; + + { + RwLock::SharedLockScope _(m_IndexLock); + + for (auto& Kv : m_Index) + { + const IoHash& Hash = Kv.first; + const DiskLocation& Loc = Kv.second; + + ZenCacheValue Value; + + if (!GetInlineCacheValue(Loc, Value)) + { + ZEN_ASSERT(Loc.IsFlagSet(DiskLocation::kStandaloneFile)); + StandaloneFiles.push_back({.Key = Hash, .Location = Loc}); + } + else + { + if (GetStandaloneCacheValue(Hash, Value, Loc)) + { + // Hash contents + + const IoHash ComputedHash = HashBuffer(Value.Value); + + if (ComputedHash != Hash) + { + BadChunks.push_back(Hash); + } + } + else + { + // Non-existent + } + } + } + } + + if (Ctx.RunRecovery()) + { + // Clean out bad chunks + } + + if (!BadChunks.empty()) + { + Ctx.ReportBadChunks(BadChunks); + } } void @@ -729,6 +821,17 @@ ZenCacheDiskLayer::Flush() } } +void +ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) +{ + RwLock::SharedLockScope _(m_Lock); + + for (auto& Kv : m_Buckets) + { + Kv.second.Scrub(Ctx); + } +} + ////////////////////////////////////////////////////////////////////////// ZenCacheTracker::ZenCacheTracker(ZenCacheStore& CacheStore) diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 2cc3abb53..f96757409 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -65,6 +65,7 @@ private: bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); void Put(const IoHash& HashKey, const ZenCacheValue& Value); + void Scrub(ScrubContext& Ctx); }; RwLock m_Lock; -- cgit v1.2.3 From 9ff7a7992ba7ab4eaf3c57bb51b24a6f7d5524c5 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 23 Sep 2021 13:56:06 +0200 Subject: cidstore: added some implementation notes --- zenstore/include/zenstore/cidstore.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index f4439e083..2eea04164 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -25,6 +25,11 @@ class IoBuffer; * be used to deal with other kinds of indirections in the future. For example, if we want * to support chunking then a CID may represent a list of chunks which could be concatenated * to form the referenced chunk. + * + * It would likely be possible to implement this mapping in a more efficient way if we + * integrate it into the CAS store itself, so we can avoid maintaining copies of large + * hashes in multiple locations. This would also allow us to consolidate commit logs etc + * which would be more resilient than the current split log scheme * */ class CidStore -- cgit v1.2.3 From 85152e23a1d64f2caabf8467572e052f296133f3 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 23 Sep 2021 13:56:48 +0200 Subject: Respect Jupiter auth token expiration time. --- zenserver/upstream/jupiter.cpp | 208 ++++++++++++++++++++++++----------------- zenserver/upstream/jupiter.h | 54 ++++++----- 2 files changed, 150 insertions(+), 112 deletions(-) diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 14da8cbcc..6eaa6423b 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -40,22 +40,32 @@ namespace detail { CloudCacheSessionState(CloudCacheClient& Client) : OwnerClient(Client) {} ~CloudCacheSessionState() {} - void Reset() + const CloudCacheAccessToken& GetAccessToken() { - std::string Auth; - OwnerClient.AcquireAccessToken(Auth); + if (!AccessToken.IsValid()) + { + AccessToken = OwnerClient.AcquireAccessToken(); + } + return AccessToken; + } + void InvalidateAccessToken() { AccessToken = {}; } + + void Reset() + { Session.SetBody({}); - Session.SetOption(cpr::Header{{"Authorization", Auth}}); + Session.SetHeader({}); + AccessToken = GetAccessToken(); } - CloudCacheClient& OwnerClient; - cpr::Session Session; + CloudCacheClient& OwnerClient; + CloudCacheAccessToken AccessToken; + cpr::Session Session; }; } // namespace detail -CloudCacheSession::CloudCacheSession(CloudCacheClient* OuterClient) : m_Log(OuterClient->Logger()), m_CacheClient(OuterClient) +CloudCacheSession::CloudCacheSession(CloudCacheClient* CacheClient) : m_Log(CacheClient->Logger()), m_CacheClient(CacheClient) { m_SessionState = m_CacheClient->AllocSessionState(); } @@ -68,16 +78,18 @@ CloudCacheSession::~CloudCacheSession() CloudCacheResult CloudCacheSession::Authenticate() { - std::string Auth; - const bool Success = m_CacheClient->AcquireAccessToken(Auth); - return {.Success = Success}; + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + return {.Success = AccessToken.IsValid()}; } CloudCacheResult CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Key) { - std::string Auth; - m_CacheClient->AcquireAccessToken(Auth); + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/c/ddc/" << m_CacheClient->DdcNamespace() << "/" << BucketId << "/" << Key << ".raw"; @@ -85,7 +97,7 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", Auth}}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); cpr::Response Response = Session.Get(); ZEN_DEBUG("GET {}", Response); @@ -94,6 +106,10 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke { return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -110,9 +126,13 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, const IoHash& Key) CloudCacheResult CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenContentType RefType) { + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + const std::string ContentType = RefType == ZenContentType::kCbObject ? "application/x-ue-cb" : "application/octet-stream"; - std::string Auth; - m_CacheClient->AcquireAccessToken(Auth); ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << m_CacheClient->BlobStoreNamespace() << "/" << BucketId << "/" @@ -121,7 +141,7 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", Auth}, {"Accept", ContentType}}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", ContentType}}); cpr::Response Response = Session.Get(); ZEN_DEBUG("GET {}", Response); @@ -130,6 +150,10 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte { return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -140,8 +164,11 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte CloudCacheResult CloudCacheSession::GetCompressedBlob(const IoHash& Key) { - std::string Auth; - m_CacheClient->AcquireAccessToken(Auth); + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); @@ -149,7 +176,7 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", Auth}, {"Accept", "application/x-ue-comp"}}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-comp"}}); cpr::Response Response = Session.Get(); ZEN_DEBUG("GET {}", Response); @@ -168,10 +195,13 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) CloudCacheResult CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Key, IoBuffer DerivedData) { - IoHash Hash = IoHash::HashBuffer(DerivedData.Data(), DerivedData.Size()); + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } - std::string Auth; - m_CacheClient->AcquireAccessToken(Auth); + IoHash Hash = IoHash::HashBuffer(DerivedData.Data(), DerivedData.Size()); ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/c/ddc/" << m_CacheClient->DdcNamespace() << "/" << BucketId << "/" << Key; @@ -179,8 +209,9 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Ke auto& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption( - cpr::Header{{"Authorization", Auth}, {"X-Jupiter-IoHash", Hash.ToHexString()}, {"Content-Type", "application/octet-stream"}}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, + {"X-Jupiter-IoHash", Hash.ToHexString()}, + {"Content-Type", "application/octet-stream"}}); Session.SetBody(cpr::Body{(const char*)DerivedData.Data(), DerivedData.Size()}); cpr::Response Response = Session.Put(); @@ -190,6 +221,10 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Ke { return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, @@ -205,11 +240,15 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, const IoHash& Key, CloudCacheResult CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType) { + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + IoHash Hash = IoHash::HashBuffer(Ref.Data(), Ref.Size()); const std::string ContentType = RefType == ZenContentType::kCbObject ? "application/x-ue-cb" : "application/octet-stream"; - std::string Auth; - m_CacheClient->AcquireAccessToken(Auth); ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << m_CacheClient->BlobStoreNamespace() << "/" << BucketId << "/" @@ -218,7 +257,8 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", Auth}, {"X-Jupiter-IoHash", Hash.ToHexString()}, {"Content-Type", ContentType}}); + Session.SetOption( + cpr::Header{{"Authorization", AccessToken.Value}, {"X-Jupiter-IoHash", Hash.ToHexString()}, {"Content-Type", ContentType}}); Session.SetBody(cpr::Body{(const char*)Ref.Data(), Ref.Size()}); cpr::Response Response = Session.Put(); @@ -228,6 +268,10 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer { return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, @@ -237,8 +281,11 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer CloudCacheResult CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) { - std::string Auth; - m_CacheClient->AcquireAccessToken(Auth); + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); @@ -246,7 +293,7 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", Auth}, {"Content-Type", "application/x-ue-comp"}}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-comp"}}); Session.SetBody(cpr::Body{(const char*)Blob.Data(), Blob.Size()}); cpr::Response Response = Session.Put(); @@ -256,6 +303,10 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) { return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, @@ -274,22 +325,21 @@ CloudCacheSession::Filter(std::string_view BucketId, const std::vector& return {}; } -////////////////////////////////////////////////////////////////////////// - -std::string -CloudCacheAccessToken::GetAuthorizationHeaderValue() +const CloudCacheAccessToken& +CloudCacheSession::GetAccessToken() { - RwLock::SharedLockScope _(m_Lock); - - return "Bearer {}"_format(m_Token); + return m_SessionState->GetAccessToken(); } -inline void -CloudCacheAccessToken::SetToken(std::string_view Token) +bool +CloudCacheSession::VerifyAccessToken(long StatusCode) { - RwLock::ExclusiveLockScope _(m_Lock); - m_Token = Token; - ++m_Serial; + if (StatusCode == 401) + { + m_SessionState->InvalidateAccessToken(); + return false; + } + return true; } ////////////////////////////////////////////////////////////////////////// @@ -354,60 +404,33 @@ CloudCacheClient::~CloudCacheClient() } } -bool -CloudCacheClient::AcquireAccessToken(std::string& AuthorizationHeaderValue) +CloudCacheAccessToken +CloudCacheClient::AcquireAccessToken() { - // TODO: check for expiration - - if (!m_IsValid) - { - ExtendableStringBuilder<128> OAuthFormData; - OAuthFormData << "client_id=" << m_OAuthClientId - << "&scope=cache_access&grant_type=client_credentials&client_secret=" << m_OAuthSecret; - - const uint32_t CurrentSerial = m_AccessToken.GetSerial(); + using namespace std::chrono; - static RwLock AuthMutex; - RwLock::ExclusiveLockScope _(AuthMutex); - - // Protect against redundant authentication operations - if (m_AccessToken.GetSerial() != CurrentSerial) - { - // TODO: this could verify that the token is actually valid and retry if not? - - return true; - } + ExtendableStringBuilder<128> OAuthFormData; + OAuthFormData << "client_id=" << m_OAuthClientId << "&scope=cache_access&grant_type=client_credentials&client_secret=" << m_OAuthSecret; - std::string data{OAuthFormData}; + std::string Body{OAuthFormData}; - cpr::Response Response = - cpr::Post(cpr::Url{m_OAuthFullUri}, cpr::Header{{"Content-Type", "application/x-www-form-urlencoded"}}, cpr::Body{data}); + cpr::Response Response = + cpr::Post(cpr::Url{m_OAuthFullUri}, cpr::Header{{"Content-Type", "application/x-www-form-urlencoded"}}, cpr::Body{Body}); - std::string Body{std::move(Response.text)}; + Body = std::move(Response.text); - // Parse JSON response - - std::string JsonError; - json11::Json JsonResponse = json11::Json::parse(Body, /* out */ JsonError); - if (!JsonError.empty()) - { - ZEN_WARN("failed to parse OAuth response: '{}'", JsonError); - - return false; - } - - std::string AccessToken = JsonResponse["access_token"].string_value(); - int ExpiryTimeSeconds = JsonResponse["expires_in"].int_value(); - ZEN_UNUSED(ExpiryTimeSeconds); - - m_AccessToken.SetToken(AccessToken); - - m_IsValid = true; + std::string JsonError; + json11::Json JsonResponse = json11::Json::parse(Body, JsonError); + if (!JsonError.empty()) + { + return {}; } - AuthorizationHeaderValue = m_AccessToken.GetAuthorizationHeaderValue(); + std::string AccessToken = std::string("Bearer ") + JsonResponse["access_token"].string_value(); + int64_t ExpiresInSeconds = static_cast(JsonResponse["expires_in"].int_value()); + steady_clock::time_point ExpireTime = steady_clock::now() + seconds(ExpiresInSeconds); - return true; + return {std::move(AccessToken), ExpireTime}; } detail::CloudCacheSessionState* @@ -434,8 +457,19 @@ CloudCacheClient::AllocSessionState() void CloudCacheClient::FreeSessionState(detail::CloudCacheSessionState* State) { + const bool IsTokenValid = State->AccessToken.IsValid(); + RwLock::ExclusiveLockScope _(m_SessionStateLock); m_SessionStateCache.push_front(State); + + // Invalidate all cached access tokens if any one fails + if (!IsTokenValid) + { + for (auto& CachedState : m_SessionStateCache) + { + CachedState->AccessToken = {}; + } + } } } // namespace zen diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index 94e7e7680..868a7b099 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -29,15 +30,17 @@ class CbObjectView; */ struct CloudCacheAccessToken { - std::string GetAuthorizationHeaderValue(); - void SetToken(std::string_view Token); + static constexpr int64_t ExpireMarginInSeconds = 30; - inline uint32_t GetSerial() const { return m_Serial.load(std::memory_order::memory_order_relaxed); } + std::string Value; + std::chrono::steady_clock::time_point ExpireTime; -private: - RwLock m_Lock; - std::string m_Token; - std::atomic m_Serial; + bool IsValid() const + { + return !Value.empty() && + ExpireMarginInSeconds < + std::chrono::duration_cast(ExpireTime - std::chrono::steady_clock::now()).count(); + } }; struct CloudCacheResult @@ -60,7 +63,7 @@ struct CloudCacheResult class CloudCacheSession { public: - CloudCacheSession(CloudCacheClient* OuterClient); + CloudCacheSession(CloudCacheClient* CacheClient); ~CloudCacheSession(); CloudCacheResult Authenticate(); @@ -77,7 +80,9 @@ public: std::vector Filter(std::string_view BucketId, const std::vector& ChunkHashes); private: - inline spdlog::logger& Log() { return m_Log; } + inline spdlog::logger& Log() { return m_Log; } + const CloudCacheAccessToken& GetAccessToken(); + bool VerifyAccessToken(long StatusCode); spdlog::logger& m_Log; RefPtr m_CacheClient; @@ -104,26 +109,25 @@ public: CloudCacheClient(const CloudCacheClientOptions& Options); ~CloudCacheClient(); - bool AcquireAccessToken(std::string& AuthorizationHeaderValue); - std::string_view DdcNamespace() const { return m_DdcNamespace; } - std::string_view BlobStoreNamespace() const { return m_BlobStoreNamespace; } - std::string_view ServiceUrl() const { return m_ServiceUrl; } - bool IsValid() const { return m_IsValid; } + CloudCacheAccessToken AcquireAccessToken(); + std::string_view DdcNamespace() const { return m_DdcNamespace; } + std::string_view BlobStoreNamespace() const { return m_BlobStoreNamespace; } + std::string_view ServiceUrl() const { return m_ServiceUrl; } + bool IsValid() const { return m_IsValid; } spdlog::logger& Logger() { return m_Log; } private: - spdlog::logger& m_Log; - std::string m_ServiceUrl; - std::string m_OAuthDomain; - std::string m_OAuthUriPath; - std::string m_OAuthFullUri; - std::string m_DdcNamespace; - std::string m_BlobStoreNamespace; - std::string m_OAuthClientId; - std::string m_OAuthSecret; - CloudCacheAccessToken m_AccessToken; - bool m_IsValid = false; + spdlog::logger& m_Log; + std::string m_ServiceUrl; + std::string m_OAuthDomain; + std::string m_OAuthUriPath; + std::string m_OAuthFullUri; + std::string m_DdcNamespace; + std::string m_BlobStoreNamespace; + std::string m_OAuthClientId; + std::string m_OAuthSecret; + bool m_IsValid = false; RwLock m_SessionStateLock; std::list m_SessionStateCache; -- cgit v1.2.3 From cbf8832318cd47d6379884d622dbceca8dea3e8c Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 23 Sep 2021 17:24:56 +0200 Subject: Simpler upstream stats. Enabled with --upstream-stats. --- zenserver/config.cpp | 7 ++ zenserver/config.h | 1 + zenserver/upstream/upstreamcache.cpp | 203 +++++++++++++++++++---------------- zenserver/upstream/upstreamcache.h | 15 +++ zenserver/zenserver.cpp | 2 + 5 files changed, 136 insertions(+), 92 deletions(-) diff --git a/zenserver/config.cpp b/zenserver/config.cpp index c21638258..457a91e49 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -214,6 +214,13 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z cxxopts::value(ServiceConfig.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), ""); + options.add_option("cache", + "", + "upstream-stats", + "Collect performance metrics for upstream endpoints", + cxxopts::value(ServiceConfig.UpstreamCacheConfig.StatsEnabled)->default_value("false"), + ""); + try { auto result = options.parse(argc, argv); diff --git a/zenserver/config.h b/zenserver/config.h index 6ade1b401..c9aa06284 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -50,6 +50,7 @@ struct ZenUpstreamCacheConfig ZenUpstreamZenConfig ZenConfig; int UpstreamThreadCount = 4; UpstreamCachePolicy CachePolicy = UpstreamCachePolicy::ReadWrite; + bool StatsEnabled = false; }; struct ZenServiceConfig diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 9d43462c0..f056c1c76 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -153,6 +153,7 @@ namespace detail { CacheRecord.IterateAttachments([&Session, &Result, &Package](CbFieldView AttachmentHash) { CloudCacheResult AttachmentResult = Session.GetCompressedBlob(AttachmentHash.AsHash()); + Result.Bytes += AttachmentResult.Bytes; Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; Result.ErrorCode = AttachmentResult.ErrorCode; @@ -176,7 +177,6 @@ namespace detail { Package.Save(Writer); Result.Response = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); - Result.Bytes = MemStream.Size(); } } } @@ -247,21 +247,26 @@ namespace detail { } else { + bool Success = false; int64_t TotalBytes = 0ull; double TotalElapsedSeconds = 0.0; for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) { - CloudCacheResult Result; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + Success = false; + for (int32_t Attempt = 0; Attempt < MaxAttempts; Attempt++) { - Result = Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); + if (CloudCacheResult Result = Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); + Result.Success) + { + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + Success = true; + break; + } } - TotalBytes += Result.Bytes; - TotalElapsedSeconds += Result.ElapsedSeconds; - - if (!Result.Success) + if (!Success) { return {.Reason = "Failed to upload payload", .Bytes = TotalBytes, @@ -270,29 +275,38 @@ namespace detail { } } - CloudCacheResult Result; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + Success = false; + for (int32_t Attempt = 0; Attempt < MaxAttempts; Attempt++) { - Result = - Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, ZenContentType::kCbObject); + if (CloudCacheResult Result = Session.PutRef(CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + RecordValue, + ZenContentType::kCbObject); + Result.Success) + { + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + Success = true; + break; + } } - TotalBytes += Result.Bytes; - TotalElapsedSeconds += Result.ElapsedSeconds; - - return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = Result.Success}; + return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = Success}; } } - catch (std::exception& e) + catch (std::exception& Err) { - return {.Reason = std::string(e.what()), .Success = false}; + return {.Reason = std::string(Err.what()), .Success = false}; } } + virtual UpstreamEndpointStats& Stats() override { return m_Stats; } + private: bool m_UseLegacyDdc; std::string m_DisplayName; RefPtr m_Client; + UpstreamEndpointStats m_Stats; std::atomic_bool m_HealthOk{false}; }; @@ -473,9 +487,12 @@ namespace detail { } } + virtual UpstreamEndpointStats& Stats() override { return m_Stats; } + private: std::string m_DisplayName; RefPtr m_Client; + UpstreamEndpointStats m_Stats; std::atomic_bool m_HealthOk{false}; }; @@ -483,87 +500,95 @@ namespace detail { ////////////////////////////////////////////////////////////////////////// -class UpstreamStats final +struct UpstreamStats { static constexpr uint64_t MaxSampleCount = 100ull; - struct StatCounters - { - int64_t Bytes = {}; - int64_t Count = {}; - double Seconds = {}; - }; - - using StatsMap = std::unordered_map; - - struct EndpointStats - { - mutable std::mutex Lock; - StatsMap Counters; - }; - -public: - UpstreamStats() : m_Log(logging::Get("upstream")) {} + UpstreamStats(bool Enabled) : m_Enabled(Enabled) {} - void Add(const UpstreamEndpoint& Endpoint, const GetUpstreamCacheResult& Result) + void Add(spdlog::logger& Logger, + UpstreamEndpoint& Endpoint, + const GetUpstreamCacheResult& Result, + const std::vector>& Endpoints) { - std::unique_lock Lock(m_DownStats.Lock); + if (!m_Enabled) + { + return; + } - auto& Counters = m_DownStats.Counters[&Endpoint]; - Counters.Bytes += Result.Bytes; - Counters.Seconds += Result.ElapsedSeconds; - Counters.Count++; + UpstreamEndpointStats& Stats = Endpoint.Stats(); + if (Result.Success) + { + Stats.HitCount++; + Stats.DownBytes.fetch_add(double(Result.Bytes) / 1024.0 / 1024.0); + Stats.SecondsDown.fetch_add(Result.ElapsedSeconds); + } + else + { + Stats.MissCount++; + } - if (Counters.Count >= MaxSampleCount) + if (m_SampleCount++ % MaxSampleCount) { - LogStats("STATS - (downstream):"sv, m_DownStats.Counters); - Counters = StatCounters{}; + Dump(Logger, Endpoints); } } - void Add(const UpstreamEndpoint& Endpoint, const PutUpstreamCacheResult& Result) + void Add(spdlog::logger& Logger, + UpstreamEndpoint& Endpoint, + const PutUpstreamCacheResult& Result, + const std::vector>& Endpoints) { - std::unique_lock Lock(m_UpStats.Lock); + if (!m_Enabled) + { + return; + } - auto& Counters = m_UpStats.Counters[&Endpoint]; - Counters.Bytes += Result.Bytes; - Counters.Seconds += Result.ElapsedSeconds; - Counters.Count++; + UpstreamEndpointStats& Stats = Endpoint.Stats(); + if (Result.Success) + { + Stats.UpCount++; + Stats.UpBytes.fetch_add(double(Result.Bytes) / 1024.0 / 1024.0); + Stats.SecondsUp.fetch_add(Result.ElapsedSeconds); + } - if (Counters.Count >= MaxSampleCount) + if (m_SampleCount++ % MaxSampleCount) { - LogStats("STATS - (upstream):"sv, m_UpStats.Counters); - Counters = StatCounters{}; + Dump(Logger, Endpoints); } } -private: - void LogStats(std::string_view What, const std::unordered_map& EndpointStats) + void Dump(spdlog::logger& Logger, const std::vector>& Endpoints) { - for (const auto& Kv : EndpointStats) + for (auto& Ep : Endpoints) { - const UpstreamEndpoint& Endpoint = *Kv.first; - const StatCounters& Counters = Kv.second; - const double TotalMb = double(Counters.Bytes) / 1024.0 / 1024.0; - - ZEN_UNUSED(Endpoint); - - ZEN_INFO("{} Endpoint: {}, Bytes: {:.2f} MB, Time: {:.2f} s, Speed: {:.2f} MB/s, Avg: {:.2f} ms/request, Samples: {}", - What, - Kv.first->DisplayName(), - TotalMb, - Counters.Seconds, - TotalMb / Counters.Seconds, - (Counters.Seconds * 1000.0) / double(Counters.Count), - Counters.Count); + // These stats will not be totally correct as the numbers are not captured atomically + + UpstreamEndpointStats& Stats = Ep->Stats(); + const uint64_t HitCount = Stats.HitCount; + const uint64_t MissCount = Stats.MissCount; + const double DownBytes = Stats.DownBytes; + const double SecondsDown = Stats.SecondsDown; + const double UpBytes = Stats.UpBytes; + const double SecondsUp = Stats.SecondsUp; + + const double UpSpeed = UpBytes > 0 ? UpBytes / SecondsUp : 0.0; + const double DownSpeed = DownBytes > 0 ? DownBytes / SecondsDown : 0.0; + const uint64_t TotalCount = HitCount + MissCount; + const double HitRate = TotalCount > 0 ? (double(HitCount) / double(TotalCount)) * 100.0 : 0.0; + + Logger.info("STATS - '{}', Hit rate: {:.2f}%, DOWN: '{:.2f} MiB {:.2f} MiB/s', UP: '{:.2f} MiB {:.2f} MiB/s'", + Ep->DisplayName(), + HitRate, + DownBytes, + DownSpeed, + UpBytes, + UpSpeed); } } - spdlog::logger& Log() { return m_Log; } - - spdlog::logger& m_Log; - EndpointStats m_UpStats; - EndpointStats m_DownStats; + bool m_Enabled; + std::atomic_uint64_t m_SampleCount = {}; }; ////////////////////////////////////////////////////////////////////////// @@ -576,6 +601,7 @@ public: , m_Options(Options) , m_CacheStore(CacheStore) , m_CidStore(CidStore) + , m_Stats(Options.StatsEnabled) { } @@ -621,9 +647,11 @@ public: { if (Endpoint->IsHealthy()) { - if (GetUpstreamCacheResult Result = Endpoint->GetCacheRecord(CacheKey, Type); Result.Success) + const GetUpstreamCacheResult Result = Endpoint->GetCacheRecord(CacheKey, Type); + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); + + if (Result.Success) { - m_Stats.Add(*Endpoint, Result); return Result; } } @@ -641,9 +669,11 @@ public: { if (Endpoint->IsHealthy()) { - if (GetUpstreamCacheResult Result = Endpoint->GetCachePayload(PayloadKey); Result.Success) + const GetUpstreamCacheResult Result = Endpoint->GetCachePayload(PayloadKey); + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); + + if (Result.Success) { - m_Stats.Add(*Endpoint, Result); return Result; } } @@ -707,18 +737,7 @@ private: if (Endpoint->IsHealthy()) { const PutUpstreamCacheResult Result = Endpoint->PutCacheRecord(CacheRecord, CacheValue.Value, std::span(Payloads)); - if (Result.Success) - { - m_Stats.Add(*Endpoint, Result); - } - else - { - ZEN_WARN("process upstream FAILED, '{}/{}' FAILED, endpoint '{}', reason: '{}'", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - Endpoint->DisplayName(), - Result.Reason); - } + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); } } } diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index 96ee8bddc..0e736480b 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -40,6 +41,7 @@ struct UpstreamCacheOptions uint32_t ThreadCount = 4; bool ReadUpstream = true; bool WriteUpstream = true; + bool StatsEnabled = false; }; enum class UpstreamStatusCode : uint8_t @@ -79,6 +81,17 @@ struct UpstreamEndpointHealth bool Ok = false; }; +struct UpstreamEndpointStats +{ + std::atomic_uint64_t HitCount = {}; + std::atomic_uint64_t MissCount = {}; + std::atomic_uint64_t UpCount = {}; + std::atomic UpBytes = {}; + std::atomic DownBytes = {}; + std::atomic SecondsUp = {}; + std::atomic SecondsDown = {}; +}; + /** * The upstream endpont is responsible for handling upload/downloading of cache records. */ @@ -100,6 +113,8 @@ public: virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, IoBuffer RecordValue, std::span Payloads) = 0; + + virtual UpstreamEndpointStats& Stats() = 0; }; /** diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index e3b61568f..7538c090f 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -188,6 +188,8 @@ public: UpstreamOptions.ThreadCount = static_cast(UpstreamConfig.UpstreamThreadCount); } + UpstreamOptions.StatsEnabled = UpstreamConfig.StatsEnabled; + UpstreamCache = zen::MakeUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore); if (!UpstreamConfig.ZenConfig.Url.empty()) -- cgit v1.2.3 From 6d55744739cf39b0ab30e5ab0cf2966ede867a22 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 23 Sep 2021 17:53:23 +0200 Subject: Added --upstream-jupiter-prod for easy production settings. --- zenserver/config.cpp | 7 +++++++ zenserver/config.h | 1 + zenserver/zenserver.cpp | 13 ++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 457a91e49..91fb80747 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -186,6 +186,13 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z cxxopts::value(ServiceConfig.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""), ""); + options.add_option("cache", + "", + "upstream-jupiter-prod", + "Enable Jupiter upstream caching using production settings", + cxxopts::value(ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseProductionSettings)->default_value("false"), + ""); + options.add_option("cache", "", "upstream-jupiter-dev", diff --git a/zenserver/config.h b/zenserver/config.h index c9aa06284..ce059bdb2 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -28,6 +28,7 @@ struct ZenUpstreamJupiterConfig std::string Namespace; std::string DdcNamespace; bool UseDevelopmentSettings = false; + bool UseProductionSettings = false; bool UseLegacyDdc = false; }; diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 7538c090f..fe4f41ab5 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -200,7 +200,18 @@ public: { zen::CloudCacheClientOptions Options; - if (UpstreamConfig.JupiterConfig.UseDevelopmentSettings) + if (UpstreamConfig.JupiterConfig.UseProductionSettings) + { + Options = zen::CloudCacheClientOptions{ + .ServiceUrl = "https://jupiter.devtools.epicgames.com"sv, + .DdcNamespace = "ue.ddc"sv, + .BlobStoreNamespace = "ue.ddc"sv, + .OAuthProvider = "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv, + .OAuthClientId = "0oao91lrhqPiAlaGD0x7"sv, + .OAuthSecret = "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv, + .UseLegacyDdc = false}; + } + else if (UpstreamConfig.JupiterConfig.UseDevelopmentSettings) { Options = zen::CloudCacheClientOptions{ .ServiceUrl = "https://jupiter.devtools-dev.epicgames.com"sv, -- cgit v1.2.3 From ef0c8e3f91b5f9fed8a8ae0547cab733e0a70de1 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 25 Sep 2021 18:40:31 +0200 Subject: Ensure FILE_RENAME_INFO structure allocation is freed also if FileCasStrategy::InsertChunk() throws --- zenstore/filecas.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index c036efd35..0b18848d5 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -133,6 +134,8 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) memcpy(RenameInfo->FileName, FileName.c_str(), FileName.size() * sizeof(WCHAR)); RenameInfo->FileName[FileName.size()] = 0; + auto $ = MakeGuard([&] { Memory::Free(RenameInfo); }); + // Try to move file into place BOOL Success = SetFileInformationByHandle(FileRef.FileHandle, FileRenameInfo, RenameInfo, BufferSize); @@ -175,8 +178,6 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) Success = SetFileInformationByHandle(FileRef.FileHandle, FileRenameInfo, RenameInfo, BufferSize); } - Memory::Free(RenameInfo); - if (Success) { return CasStore::InsertResult{.New = true}; -- cgit v1.2.3 From e96d3f99687cdd57d6d2b342b465928c72e84c27 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 25 Sep 2021 21:21:16 +0200 Subject: Added TemporaryFile implementation, provides a simple abstraction around temporary files --- zenstore/basicfile.cpp | 84 ++++++++++++++++++++++++++++++++++- zenstore/include/zenstore/basicfile.h | 38 +++++++++++++++- 2 files changed, 119 insertions(+), 3 deletions(-) diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index fe54184cf..233efa11c 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -36,11 +36,16 @@ void BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec) { const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; - const DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; const DWORD dwShareMode = FILE_SHARE_READ; const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; HANDLE hTemplateFile = nullptr; + if (IsCreate) + { + dwDesiredAccess |= DELETE; + } + HANDLE FileHandle = CreateFile(FileName.c_str(), dwDesiredAccess, dwShareMode, @@ -63,6 +68,7 @@ BasicFile::Close() if (m_FileHandle) { ::CloseHandle(m_FileHandle); + m_FileHandle = nullptr; } } @@ -158,6 +164,51 @@ BasicFile::FileSize() return uint64_t(liFileSize.QuadPart); } +////////////////////////////////////////////////////////////////////////// + +TemporaryFile::~TemporaryFile() +{ + Close(); +} + +void +TemporaryFile::Close() +{ + if (m_FileHandle) + { + // Mark file for deletion when final handle is closed + + FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; + + SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); + + BasicFile::Close(); + } +} + +void +TemporaryFile::CreateTemporary(std::filesystem::path TempDirName, std::error_code& Ec) +{ + StringBuilder<64> TempName; + Oid::NewOid().ToString(TempName); + + m_TempPath = TempDirName / TempName.c_str(); + + const bool IsCreate = true; + + Open(m_TempPath, IsCreate, Ec); +} + +void +TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec) +{ + // We intentionally call the base class Close() since otherwise we'll end up + // deleting the temporary file + BasicFile::Close(); + + std::filesystem::rename(m_TempPath, FinalFileName, Ec); +} + #if ZEN_WITH_TESTS TEST_CASE("BasicFile") @@ -171,6 +222,37 @@ TEST_CASE("BasicFile") CHECK(File1.FileSize() == 4); } +TEST_CASE("TemporaryFile") +{ + ScopedCurrentDirectoryChange _; + + SUBCASE("DeleteOnClose") + { + TemporaryFile TmpFile; + std::error_code Ec; + TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); + CHECK(!Ec); + CHECK(std::filesystem::exists(TmpFile.GetPath())); + TmpFile.Close(); + CHECK(std::filesystem::exists(TmpFile.GetPath()) == false); + } + + SUBCASE("MoveIntoPlace") + { + TemporaryFile TmpFile; + std::error_code Ec; + TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); + CHECK(!Ec); + std::filesystem::path TempPath = TmpFile.GetPath(); + std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; + CHECK(std::filesystem::exists(TempPath)); + TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); + CHECK(!Ec); + CHECK(std::filesystem::exists(TempPath) == false); + CHECK(std::filesystem::exists(FinalPath)); + } +} + void basicfile_forcelink() { diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index d4d65b366..8a11ea98a 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -26,6 +26,10 @@ class BasicFile public: BasicFile() = default; ~BasicFile(); + + BasicFile(const BasicFile&) = delete; + BasicFile& operator=(const BasicFile&) = delete; + void Open(std::filesystem::path FileName, bool IsCreate); void Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec); void Close(); @@ -35,13 +39,43 @@ public: void Write(const void* Data, uint64_t Size, uint64_t FileOffset); void Flush(); uint64_t FileSize(); - void* Handle() { return m_FileHandle; } IoBuffer ReadAll(); -private: + inline void* Handle() { return m_FileHandle; } + +protected: void* m_FileHandle = nullptr; // This is either null or valid }; +/** + * Simple abstraction for a temporary file + * + * Works like a regular BasicFile but implements a simple mechanism to allow creating + * a temporary file for writing in a directory which may later be moved atomically + * into the intended location after it has been fully written to. + * + */ + +class TemporaryFile : public BasicFile +{ +public: + TemporaryFile() = default; + ~TemporaryFile(); + + TemporaryFile(const TemporaryFile&) = delete; + TemporaryFile& operator=(const TemporaryFile&) = delete; + + void Close(); + void CreateTemporary(std::filesystem::path TempDirName, std::error_code& Ec); + void MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec); + const std::filesystem::path& GetPath() const { return m_TempPath; } + +private: + std::filesystem::path m_TempPath; + + using BasicFile::Open; +}; + ZENCORE_API void basicfile_forcelink(); } // namespace zen -- cgit v1.2.3 From cd3386a73371e041fe7f392c8fbb890d25c670f2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 25 Sep 2021 21:22:27 +0200 Subject: Removed some unnecessary filesystem wrapper functions which accepted wchar* arguments Also moved some platform specific functionality into Windows conditional sections --- zencore/filesystem.cpp | 30 +++++++++++------------------- zencore/include/zencore/filesystem.h | 3 --- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 45e177aaa..f6e410ee2 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -21,6 +21,7 @@ namespace zen { using namespace std::literals; +#if ZEN_PLATFORM_WINDOWS static bool DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) { @@ -47,18 +48,6 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) return false; } -bool -CreateDirectories(const wchar_t* Dir) -{ - return std::filesystem::create_directories(Dir); -} - -bool -CreateDirectories(const std::filesystem::path& Dir) -{ - return std::filesystem::create_directories(Dir); -} - // Erase all files and directories in a given directory, leaving an empty directory // behind @@ -137,11 +126,12 @@ WipeDirectory(const wchar_t* DirPath) return true; } +#endif bool -DeleteDirectories(const wchar_t* DirPath) +CreateDirectories(const std::filesystem::path& Dir) { - return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE; + return std::filesystem::create_directories(Dir); } bool @@ -151,16 +141,14 @@ CleanDirectory(const wchar_t* DirPath) { return WipeDirectory(DirPath); } - else - { - return CreateDirectories(DirPath); - } + + return CreateDirectories(DirPath); } bool DeleteDirectories(const std::filesystem::path& Dir) { - return DeleteDirectories(Dir.c_str()); + return WipeDirectory(Dir.c_str()) && RemoveDirectoryW(Dir.c_str()) == TRUE; } bool @@ -174,6 +162,7 @@ CleanDirectory(const std::filesystem::path& Dir) bool SupportsBlockRefCounting(std::filesystem::path Path) { +#if ZEN_PLATFORM_WINDOWS ATL::CHandle Handle(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, @@ -200,6 +189,9 @@ SupportsBlockRefCounting(std::filesystem::path Path) } return true; +#else + return false; +#endif } bool diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h index a2d368d6f..16d6ede53 100644 --- a/zencore/include/zencore/filesystem.h +++ b/zencore/include/zencore/filesystem.h @@ -14,19 +14,16 @@ class IoBuffer; /** Delete directory (after deleting any contents) */ -ZENCORE_API bool DeleteDirectories(const wchar_t* dir); ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); /** Ensure directory exists. Will also create any required parent directories */ -ZENCORE_API bool CreateDirectories(const wchar_t* dir); ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectory(const wchar_t* dir); ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); /** Map native file handle to a path -- cgit v1.2.3 From eb5038d1f271965074bdafb2a581a0fdf099ea0e Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 21:02:31 +0200 Subject: Removed accidental x86 configurations --- zen.sln | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/zen.sln b/zen.sln index 052c030ae..dec613b24 100644 --- a/zen.sln +++ b/zen.sln @@ -53,73 +53,49 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 Release|x64 = Release|x64 - Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {D75BF9AB-C61E-4FFF-AD59-1563430F05E2}.Debug|x64.ActiveCfg = Debug|x64 {D75BF9AB-C61E-4FFF-AD59-1563430F05E2}.Debug|x64.Build.0 = Debug|x64 - {D75BF9AB-C61E-4FFF-AD59-1563430F05E2}.Debug|x86.ActiveCfg = Debug|x64 {D75BF9AB-C61E-4FFF-AD59-1563430F05E2}.Release|x64.ActiveCfg = Release|x64 {D75BF9AB-C61E-4FFF-AD59-1563430F05E2}.Release|x64.Build.0 = Release|x64 - {D75BF9AB-C61E-4FFF-AD59-1563430F05E2}.Release|x86.ActiveCfg = Release|x64 {C00173DF-B76E-4989-B576-FE2B780B2580}.Debug|x64.ActiveCfg = Debug|x64 {C00173DF-B76E-4989-B576-FE2B780B2580}.Debug|x64.Build.0 = Debug|x64 - {C00173DF-B76E-4989-B576-FE2B780B2580}.Debug|x86.ActiveCfg = Debug|x64 {C00173DF-B76E-4989-B576-FE2B780B2580}.Release|x64.ActiveCfg = Release|x64 {C00173DF-B76E-4989-B576-FE2B780B2580}.Release|x64.Build.0 = Release|x64 - {C00173DF-B76E-4989-B576-FE2B780B2580}.Release|x86.ActiveCfg = Release|x64 {8398D81C-B1B6-4327-82B1-06EACB8A144F}.Debug|x64.ActiveCfg = Debug|x64 {8398D81C-B1B6-4327-82B1-06EACB8A144F}.Debug|x64.Build.0 = Debug|x64 - {8398D81C-B1B6-4327-82B1-06EACB8A144F}.Debug|x86.ActiveCfg = Debug|x64 {8398D81C-B1B6-4327-82B1-06EACB8A144F}.Release|x64.ActiveCfg = Release|x64 {8398D81C-B1B6-4327-82B1-06EACB8A144F}.Release|x64.Build.0 = Release|x64 - {8398D81C-B1B6-4327-82B1-06EACB8A144F}.Release|x86.ActiveCfg = Release|x64 {CA7B9E04-A2D3-4A39-A7D7-FB156A2C6A48}.Debug|x64.ActiveCfg = Debug|x64 {CA7B9E04-A2D3-4A39-A7D7-FB156A2C6A48}.Debug|x64.Build.0 = Debug|x64 - {CA7B9E04-A2D3-4A39-A7D7-FB156A2C6A48}.Debug|x86.ActiveCfg = Debug|x64 {CA7B9E04-A2D3-4A39-A7D7-FB156A2C6A48}.Release|x64.ActiveCfg = Release|x64 {CA7B9E04-A2D3-4A39-A7D7-FB156A2C6A48}.Release|x64.Build.0 = Release|x64 - {CA7B9E04-A2D3-4A39-A7D7-FB156A2C6A48}.Release|x86.ActiveCfg = Release|x64 {2563249E-E695-4CC4-8FFA-335D07680C9D}.Debug|x64.ActiveCfg = Debug|x64 {2563249E-E695-4CC4-8FFA-335D07680C9D}.Debug|x64.Build.0 = Debug|x64 - {2563249E-E695-4CC4-8FFA-335D07680C9D}.Debug|x86.ActiveCfg = Debug|x64 {2563249E-E695-4CC4-8FFA-335D07680C9D}.Release|x64.ActiveCfg = Release|x64 {2563249E-E695-4CC4-8FFA-335D07680C9D}.Release|x64.Build.0 = Release|x64 - {2563249E-E695-4CC4-8FFA-335D07680C9D}.Release|x86.ActiveCfg = Release|x64 {26CBBAEB-14C1-4EFC-877D-80F48215651C}.Debug|x64.ActiveCfg = Debug|x64 {26CBBAEB-14C1-4EFC-877D-80F48215651C}.Debug|x64.Build.0 = Debug|x64 - {26CBBAEB-14C1-4EFC-877D-80F48215651C}.Debug|x86.ActiveCfg = Debug|x64 {26CBBAEB-14C1-4EFC-877D-80F48215651C}.Release|x64.ActiveCfg = Release|x64 {26CBBAEB-14C1-4EFC-877D-80F48215651C}.Release|x64.Build.0 = Release|x64 - {26CBBAEB-14C1-4EFC-877D-80F48215651C}.Release|x86.ActiveCfg = Release|x64 {77F8315D-B21D-4DB0-9A6F-2D3359F88A70}.Debug|x64.ActiveCfg = Debug|x64 {77F8315D-B21D-4DB0-9A6F-2D3359F88A70}.Debug|x64.Build.0 = Debug|x64 - {77F8315D-B21D-4DB0-9A6F-2D3359F88A70}.Debug|x86.ActiveCfg = Debug|x64 {77F8315D-B21D-4DB0-9A6F-2D3359F88A70}.Release|x64.ActiveCfg = Release|x64 {77F8315D-B21D-4DB0-9A6F-2D3359F88A70}.Release|x64.Build.0 = Release|x64 - {77F8315D-B21D-4DB0-9A6F-2D3359F88A70}.Release|x86.ActiveCfg = Release|x64 {7FFC7E77-D038-44E9-8D84-41918C355F29}.Debug|x64.ActiveCfg = Debug|x64 {7FFC7E77-D038-44E9-8D84-41918C355F29}.Debug|x64.Build.0 = Debug|x64 - {7FFC7E77-D038-44E9-8D84-41918C355F29}.Debug|x86.ActiveCfg = Debug|Win32 - {7FFC7E77-D038-44E9-8D84-41918C355F29}.Debug|x86.Build.0 = Debug|Win32 {7FFC7E77-D038-44E9-8D84-41918C355F29}.Release|x64.ActiveCfg = Release|x64 {7FFC7E77-D038-44E9-8D84-41918C355F29}.Release|x64.Build.0 = Release|x64 - {7FFC7E77-D038-44E9-8D84-41918C355F29}.Release|x86.ActiveCfg = Release|Win32 - {7FFC7E77-D038-44E9-8D84-41918C355F29}.Release|x86.Build.0 = Release|Win32 {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Debug|x64.ActiveCfg = Debug|x64 {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Debug|x64.Build.0 = Debug|x64 - {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Debug|x86.ActiveCfg = Debug|x64 {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x64.ActiveCfg = Release|x64 {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x64.Build.0 = Release|x64 - {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x86.ActiveCfg = Release|x64 {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x64.ActiveCfg = Debug|x64 {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x64.Build.0 = Debug|x64 - {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x86.ActiveCfg = Debug|x64 {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x64.ActiveCfg = Release|x64 {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x64.Build.0 = Release|x64 - {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x86.ActiveCfg = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE -- cgit v1.2.3 From 552bad5ee4bc2a89a5d1d390cc66ac30a3d6f6a1 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 21:03:42 +0200 Subject: Use /MP on all projects --- zentest-appstub/zentest-appstub.vcxproj | 1 + 1 file changed, 1 insertion(+) diff --git a/zentest-appstub/zentest-appstub.vcxproj b/zentest-appstub/zentest-appstub.vcxproj index efbe86b47..16767f418 100644 --- a/zentest-appstub/zentest-appstub.vcxproj +++ b/zentest-appstub/zentest-appstub.vcxproj @@ -124,6 +124,7 @@ true stdcpplatest MultiThreadedDebug + true Console -- cgit v1.2.3 From 4f1b3600b85461e7817d23f46eedcae40b99c160 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 21:06:06 +0200 Subject: Various BasicFile improvements * BasicFile::Open clears error_code on entry, and exits early on failure to avoid initialization of m_FileHandle * Made BasicFile::Read handle large reads * Made BasicFile::Write handle large writes * Added BasicFile::WriteAll which may be optimized in the future to handle what is essentially a file copy more efficiently --- zenstore/basicfile.cpp | 110 +++++++++++++++++++++++++++------- zenstore/include/zenstore/basicfile.h | 2 + 2 files changed, 90 insertions(+), 22 deletions(-) diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index 233efa11c..f41f04101 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -35,6 +35,8 @@ BasicFile::Open(std::filesystem::path FileName, bool IsCreate) void BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec) { + Ec.clear(); + const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; const DWORD dwShareMode = FILE_SHARE_READ; @@ -57,6 +59,8 @@ BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& if (FileHandle == INVALID_HANDLE_VALUE) { Ec = zen::MakeErrorCodeFromLastError(); + + return; } m_FileHandle = FileHandle; @@ -73,21 +77,32 @@ BasicFile::Close() } void -BasicFile::Read(void* Data, uint64_t Size, uint64_t Offset) +BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { - OVERLAPPED Ovl{}; + const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; - Ovl.Offset = DWORD(Offset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(Offset >> 32); + while (BytesToRead) + { + const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); - DWORD dwNumberOfBytesToRead = gsl::narrow(Size); - DWORD dwNumberOfBytesRead = 0; + OVERLAPPED Ovl{}; - BOOL Success = ::ReadFile(m_FileHandle, Data, dwNumberOfBytesToRead, &dwNumberOfBytesRead, &Ovl); + Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); + Ovl.OffsetHigh = DWORD(FileOffset >> 32); - if (!Success) - { - ThrowLastError("Failed to read from file '{}'"_format(zen::PathFromHandle(m_FileHandle))); + DWORD dwNumberOfBytesRead = 0; + BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); + + ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead); + + if (!Success) + { + ThrowLastError("Failed to read from file '{}'"_format(zen::PathFromHandle(m_FileHandle))); + } + + BytesToRead -= NumberOfBytesToRead; + FileOffset += NumberOfBytesToRead; + Data = reinterpret_cast(Data) + NumberOfBytesToRead; } } @@ -95,9 +110,7 @@ IoBuffer BasicFile::ReadAll() { IoBuffer Buffer(FileSize()); - - Read((void*)Buffer.Data(), Buffer.Size(), 0); - + Read(Buffer.MutableData(), Buffer.Size(), 0); return Buffer; } @@ -131,24 +144,56 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function> 32); + + DWORD dwNumberOfBytesWritten = 0; + + BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); - Ovl.Offset = DWORD(Offset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(Offset >> 32); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); - DWORD dwNumberOfBytesToWrite = gsl::narrow(Size); - DWORD dwNumberOfBytesWritten = 0; + return; + } - BOOL Success = ::WriteFile(m_FileHandle, Data, dwNumberOfBytesToWrite, &dwNumberOfBytesWritten, &Ovl); + Size -= NumberOfBytesToWrite; + FileOffset += NumberOfBytesToWrite; + Data = reinterpret_cast(Data) + NumberOfBytesToWrite; + } +} - if (!Success) +void +BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset) +{ + std::error_code Ec; + Write(Data, Size, Offset, Ec); + + if (Ec) { - ThrowLastError("Failed to write to file '{}'"_format(zen::PathFromHandle(m_FileHandle))); + throw std::system_error(Ec, "Failed to write to file '{}'"_format(zen::PathFromHandle(m_FileHandle))); } } +void +BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec) +{ + Write(Data.Data(), Data.Size(), 0, Ec); +} + void BasicFile::Flush() { @@ -209,6 +254,15 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std:: std::filesystem::rename(m_TempPath, FinalFileName, Ec); } +/* + ___________ __ + \__ ___/___ _______/ |_ ______ + | |_/ __ \ / ___/\ __\/ ___/ + | |\ ___/ \___ \ | | \___ \ + |____| \___ >____ > |__| /____ > + \/ \/ \/ +*/ + #if ZEN_WITH_TESTS TEST_CASE("BasicFile") @@ -220,6 +274,18 @@ TEST_CASE("BasicFile") CHECK_NOTHROW(File1.Open("zonk", true)); CHECK_NOTHROW(File1.Write("abcd", 4, 0)); CHECK(File1.FileSize() == 4); + { + IoBuffer Data = File1.ReadAll(); + CHECK(Data.Size() == 4); + CHECK_EQ(memcmp(Data.Data(), "abcd", 4), 0); + } + CHECK_NOTHROW(File1.Write("efgh", 4, 2)); + CHECK(File1.FileSize() == 6); + { + IoBuffer Data = File1.ReadAll(); + CHECK(Data.Size() == 6); + CHECK_EQ(memcmp(Data.Data(), "abefgh", 6), 0); + } } TEST_CASE("TemporaryFile") diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index 8a11ea98a..fad4a33e1 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -37,9 +37,11 @@ public: void StreamFile(std::function&& ChunkFun); void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); + void Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec); void Flush(); uint64_t FileSize(); IoBuffer ReadAll(); + void WriteAll(IoBuffer Data, std::error_code& Ec); inline void* Handle() { return m_FileHandle; } -- cgit v1.2.3 From d18c48de7e4be85d761246d445d32585060ffe3c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 21:06:23 +0200 Subject: Added some more context to http failure logging --- zenhttp/httpsys.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 28a9f7abf..9c747d17e 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -553,7 +553,10 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) CancelThreadpoolIo(Iocp); - ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}'", GetWindowsErrorAsString(SendResult), HttpReq->pRawUrl); + ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}', request id: {}", + GetWindowsErrorAsString(SendResult), + HttpReq->pRawUrl, + HttpReq->RequestId); ErrorCode = MakeWin32ErrorCode(SendResult); } -- cgit v1.2.3 From a95e301f25e9387714043bc3d5576ab7318285e6 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 21:06:54 +0200 Subject: Eliminated use of ATL in StructuredCacheStore implementation --- zenserver/cache/structuredcachestore.cpp | 80 ++++++++++++++++---------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 4f16b81f2..5e93ebaa9 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -22,8 +22,6 @@ #include #include -#include - ////////////////////////////////////////////////////////////////////////// namespace zen { @@ -292,10 +290,6 @@ struct ZenCacheDiskLayer::CacheBucket static bool Delete(std::filesystem::path BucketDir); bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); - - bool GetStandaloneCacheValue(const IoHash& HashKey, ZenCacheValue& OutValue, const DiskLocation& Loc); - - bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); void Put(const IoHash& HashKey, const ZenCacheValue& Value); void Drop(); void Flush(); @@ -310,15 +304,19 @@ private: bool m_Ok = false; uint64_t m_LargeObjectThreshold = 64 * 1024; + // These files are used to manage storage of small objects for this bucket + BasicFile m_SobsFile; TCasLogFile m_SlogFile; - void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey); - void PutLargeObject(const IoHash& HashKey, const ZenCacheValue& Value); - RwLock m_IndexLock; tsl::robin_map m_Index; uint64_t m_WriteCursor = 0; + + void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey); + void PutLargeObject(const IoHash& HashKey, const ZenCacheValue& Value); + bool GetStandaloneCacheValue(const IoHash& HashKey, ZenCacheValue& OutValue, const DiskLocation& Loc); + bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); }; ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas) @@ -353,27 +351,24 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir) std::filesystem::path SobsPath{m_BucketDir / "zen.sobs"}; std::filesystem::path SlogPath{m_BucketDir / "zen.slog"}; - CAtlFile ManifestFile; + BasicFile ManifestFile; // Try opening existing manifest file first bool IsNew = false; - HRESULT hRes = ManifestFile.Create(ManifestPath.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, OPEN_EXISTING); + std::error_code Ec; + ManifestFile.Open(ManifestPath, /* IsCreate */ false, Ec); - if (SUCCEEDED(hRes)) + if (!Ec) { - ULONGLONG FileSize; - ManifestFile.GetSize(FileSize); + uint64_t FileSize = ManifestFile.FileSize(); if (FileSize == sizeof(Oid)) { - hRes = ManifestFile.Read(&m_BucketId, sizeof(Oid)); + ManifestFile.Read(&m_BucketId, sizeof(Oid), 0); - if (SUCCEEDED(hRes)) - { - m_Ok = true; - } + m_Ok = true; } if (!m_Ok) @@ -386,16 +381,16 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir) { // No manifest file found, this is a new bucket - hRes = ManifestFile.Create(ManifestPath.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); + ManifestFile.Open(ManifestPath, /* IsCreate */ true, Ec); - if (FAILED(hRes)) + if (Ec) { - ThrowLastError("Failed to create bucket manifest '{}'"_format(ManifestPath)); + throw std::system_error(Ec, "Failed to create bucket manifest '{}'"_format(ManifestPath)); } m_BucketId.Generate(); - hRes = ManifestFile.Write(&m_BucketId, sizeof(Oid)); + ManifestFile.Write(&m_BucketId, sizeof(Oid), /* FileOffset */ 0); IsNew = true; } @@ -565,7 +560,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx) { std::vector StandaloneFiles; - std::vector BadChunks; + std::vector BadChunks; std::vector BadStandaloneChunks; { @@ -621,35 +616,38 @@ ZenCacheDiskLayer::CacheBucket::PutLargeObject(const IoHash& HashKey, const ZenC WideStringBuilder<128> DataFilePath; BuildPath(DataFilePath, HashKey); - // TODO: replace this process with a more efficient implementation with proper atomic rename - // and also avoid creating directories if we can - - std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path(); - CreateDirectories(ParentPath); + TemporaryFile DataFile; - CAtlTemporaryFile DataFile; + std::error_code Ec; + DataFile.CreateTemporary(m_BucketDir.c_str(), Ec); - HRESULT hRes = DataFile.Create(m_BucketDir.c_str()); - - if (FAILED(hRes)) + if (Ec) { - ThrowSystemException(hRes, "Failed to open temporary file for put at '{}'"_format(m_BucketDir)); + throw std::system_error(Ec, "Failed to open temporary file for put at '{}'"_format(m_BucketDir)); } - hRes = DataFile.Write(Value.Value.Data(), gsl::narrow(Value.Value.Size())); + DataFile.WriteAll(Value.Value, Ec); - if (FAILED(hRes)) + if (Ec) { - ThrowSystemException(hRes, "Failed to write payload ({} bytes) to file"_format(NiceBytes(Value.Value.Size()))); + throw std::system_error(Ec, "Failed to write payload ({} bytes) to file"_format(NiceBytes(Value.Value.Size()))); } - // Move file into place (note: not fully atomic!) + // Move file into place (atomically) - hRes = DataFile.Close(DataFilePath.c_str()); + DataFile.MoveTemporaryIntoPlace(DataFilePath.c_str(), Ec); - if (FAILED(hRes)) + if (Ec) { - ThrowSystemException(hRes, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath))); + std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path(); + CreateDirectories(ParentPath); + + DataFile.MoveTemporaryIntoPlace(DataFilePath.c_str(), Ec); + + if (Ec) + { + throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath))); + } } // Update index -- cgit v1.2.3 From 71367d759aea47588225537ae4b8ab93c461bfe6 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 22:05:16 +0200 Subject: Removed unused KV cache related code --- zenserver/cache/cachestore.cpp | 252 ------------------------------------ zenserver/cache/cachestore.h | 84 ------------ zenserver/zenserver.vcxproj | 2 - zenserver/zenserver.vcxproj.filters | 6 - 4 files changed, 344 deletions(-) delete mode 100644 zenserver/cache/cachestore.cpp delete mode 100644 zenserver/cache/cachestore.h diff --git a/zenserver/cache/cachestore.cpp b/zenserver/cache/cachestore.cpp deleted file mode 100644 index 2fc253a07..000000000 --- a/zenserver/cache/cachestore.cpp +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "cachestore.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include - -using namespace zen; -using namespace fmt::literals; - -namespace UE { - -struct CorruptionTrailer -{ - enum - { - /** Arbitrary number used to identify corruption **/ - MagicConstant = 0x1e873d89 - }; - - uint32_t Magic = MagicConstant; - uint32_t Version = 1; - uint32_t CRCofPayload = 0; - uint32_t SizeOfPayload = 0; - - void Initialize(const void* Data, size_t Size) - { - CRCofPayload = zen::MemCrc32_Deprecated(Data, Size); - SizeOfPayload = (uint32_t)Size; - } -}; - -std::filesystem::path -GenerateDdcPath(std::string_view Key, std::filesystem::path& rootDir) -{ - std::filesystem::path FilePath = rootDir; - - std::string k8{Key}; - for (auto& c : k8) - c = (char)toupper(c); - - const uint32_t Hash = zen::StrCrc_Deprecated(k8.c_str()); - - std::wstring DirName; - - DirName = u'0' + ((Hash / 100) % 10); - FilePath /= DirName; - DirName = u'0' + ((Hash / 10) % 10); - FilePath /= DirName; - DirName = u'0' + (Hash % 10); - FilePath /= DirName; - - FilePath /= Key; - - auto NativePath = FilePath.native(); - NativePath.append(L".udd"); - - return NativePath; -} - -} // namespace UE - -////////////////////////////////////////////////////////////////////////// - -FileCacheStore::FileCacheStore(const char* RootDir, const char* ReadRootDir) -{ - // Ensure root directory exists - create if it doesn't exist already - - ZEN_INFO("Initializing FileCacheStore at '{}'", std::string_view(RootDir)); - - m_RootDir = RootDir; - - std::error_code ErrorCode; - - std::filesystem::create_directories(m_RootDir, ErrorCode); - - if (ErrorCode) - { - ExtendableStringBuilder<256> Name; - WideToUtf8(m_RootDir.c_str(), Name); - - ZEN_ERROR("Could not open file cache directory '{}' for writing ({})", Name.c_str(), ErrorCode.message()); - - m_IsOk = false; - } - - if (ReadRootDir) - { - m_ReadRootDir = ReadRootDir; - - if (std::filesystem::exists(m_ReadRootDir, ErrorCode)) - { - ZEN_INFO("FileCacheStore will use additional read tree at '{}'", std::string_view(ReadRootDir)); - - m_ReadRootIsValid = true; - } - } -} - -FileCacheStore::~FileCacheStore() -{ -} - -bool -FileCacheStore::Get(std::string_view Key, CacheValue& OutValue) -{ - CAtlFile File; - - std::filesystem::path NativePath; - - HRESULT hRes = E_FAIL; - - if (m_ReadRootDir.empty() == false) - { - NativePath = UE::GenerateDdcPath(Key, m_ReadRootDir); - - hRes = File.Create(NativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); - } - - if (FAILED(hRes)) - { - NativePath = UE::GenerateDdcPath(Key, m_RootDir); - - hRes = File.Create(NativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); - } - - if (FAILED(hRes)) - { - ZEN_DEBUG("GET MISS {}", Key); - - return false; - } - - ULONGLONG FileSize; - File.GetSize(FileSize); - - if (FileSize <= 16) - { - return false; - } - - FileSize -= 16; // CorruptionWrapper trailer - - OutValue.Value = IoBuffer(IoBuffer::File, File.Detach(), 0, FileSize); - - ZEN_DEBUG("GET HIT {}", Key); - - return true; -} - -void -FileCacheStore::Put(std::string_view Key, const CacheValue& Value) -{ - const void* Data = Value.Value.Data(); - size_t Size = Value.Value.Size(); - - UE::CorruptionTrailer Trailer; - Trailer.Initialize(Data, Size); - - std::filesystem::path NativePath = UE::GenerateDdcPath(Key, m_RootDir); - - CAtlTemporaryFile File; - - ZEN_DEBUG("PUT {}", Key); - - HRESULT hRes = File.Create(m_RootDir.c_str()); - - if (SUCCEEDED(hRes)) - { - const uint8_t* WritePointer = reinterpret_cast(Data); - - while (Size) - { - const int MaxChunkSize = 16 * 1024 * 1024; - const int ChunkSize = (int)((Size > MaxChunkSize) ? MaxChunkSize : Size); - - DWORD BytesWritten = 0; - File.Write(WritePointer, ChunkSize, &BytesWritten); - - Size -= BytesWritten; - WritePointer += BytesWritten; - } - - File.Write(&Trailer, sizeof Trailer); - hRes = File.Close(NativePath.c_str()); // This renames the file to its final name - - if (FAILED(hRes)) - { - ZEN_WARN("Failed to rename temp file for key '{}' - deleting temporary file", Key); - - if (!DeleteFile(File.TempFileName())) - { - ZEN_WARN("Temp file for key '{}' could not be deleted - no value persisted", Key); - } - } - } -} - -////////////////////////////////////////////////////////////////////////// - -MemoryCacheStore::MemoryCacheStore() -{ -} - -MemoryCacheStore::~MemoryCacheStore() -{ -} - -bool -MemoryCacheStore::Get(std::string_view InKey, CacheValue& OutValue) -{ - RwLock::SharedLockScope _(m_Lock); - - auto it = m_CacheMap.find(std::string(InKey)); - - if (it == m_CacheMap.end()) - { - return false; - } - else - { - OutValue.Value = it->second; - - return true; - } -} - -void -MemoryCacheStore::Put(std::string_view Key, const CacheValue& Value) -{ - RwLock::ExclusiveLockScope _(m_Lock); - m_CacheMap[std::string(Key)] = Value.Value; -} diff --git a/zenserver/cache/cachestore.h b/zenserver/cache/cachestore.h deleted file mode 100644 index 89c6396b8..000000000 --- a/zenserver/cache/cachestore.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace zen { - -class WideStringBuilderBase; -class CasStore; - -} // namespace zen - -struct CacheValue -{ - zen::IoBuffer Value; -}; - -/****************************************************************************** - - /$$ /$$/$$ /$$ /$$$$$$ /$$ - | $$ /$$| $$ | $$ /$$__ $$ | $$ - | $$ /$$/| $$ | $$ | $$ \__/ /$$$$$$ /$$$$$$| $$$$$$$ /$$$$$$ - | $$$$$/ | $$ / $$/ | $$ |____ $$/$$_____| $$__ $$/$$__ $$ - | $$ $$ \ $$ $$/ | $$ /$$$$$$| $$ | $$ \ $| $$$$$$$$ - | $$\ $$ \ $$$/ | $$ $$/$$__ $| $$ | $$ | $| $$_____/ - | $$ \ $$ \ $/ | $$$$$$| $$$$$$| $$$$$$| $$ | $| $$$$$$$ - |__/ \__/ \_/ \______/ \_______/\_______|__/ |__/\_______/ - - Basic Key-Value cache. No restrictions on keys, and values are always opaque - binary blobs. - -******************************************************************************/ - -class CacheStore -{ -public: - virtual bool Get(std::string_view Key, CacheValue& OutValue) = 0; - virtual void Put(std::string_view Key, const CacheValue& Value) = 0; -}; - -/** File system based implementation - - Emulates the behaviour of UE4 with regards to file system structure, - and also adds a file corruption trailer to remain compatible with - the file-system based implementation (this should be made configurable) - - */ -class FileCacheStore : public CacheStore -{ -public: - FileCacheStore(const char* RootDir, const char* ReadRootDir = nullptr); - ~FileCacheStore(); - - virtual bool Get(std::string_view Key, CacheValue& OutValue) override; - virtual void Put(std::string_view Key, const CacheValue& Value) override; - -private: - std::filesystem::path m_RootDir; - std::filesystem::path m_ReadRootDir; - bool m_IsOk = true; - bool m_ReadRootIsValid = false; -}; - -class MemoryCacheStore : public CacheStore -{ -public: - MemoryCacheStore(); - ~MemoryCacheStore(); - - virtual bool Get(std::string_view Key, CacheValue& OutValue) override; - virtual void Put(std::string_view Key, const CacheValue& Value) override; - -private: - zen::RwLock m_Lock; - std::unordered_map m_CacheMap; -}; diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index 1671d98a6..29436d840 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -115,7 +115,6 @@ - @@ -138,7 +137,6 @@ - diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters index c51a8eb76..6b99ca8d7 100644 --- a/zenserver/zenserver.vcxproj.filters +++ b/zenserver/zenserver.vcxproj.filters @@ -9,9 +9,6 @@ cache - - cache - diag @@ -50,9 +47,6 @@ cache - - cache - experimental -- cgit v1.2.3 From 957840565c5970e9bede4c7f506edbad80893c47 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 22:05:49 +0200 Subject: Added HttpVerb ToString() function for use in logging --- zenhttp/httpserver.cpp | 24 ++++++++++++++++++++++++ zenhttp/include/zenhttp/httpcommon.h | 2 ++ 2 files changed, 26 insertions(+) diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp index 599c99a18..d999f7f60 100644 --- a/zenhttp/httpserver.cpp +++ b/zenhttp/httpserver.cpp @@ -128,6 +128,30 @@ HttpContentType (*ParseContentType)(const std::string_view& ContentTypeString) = ////////////////////////////////////////////////////////////////////////// +const std::string_view +ToString(HttpVerb Verb) +{ + switch (Verb) + { + case HttpVerb::kGet: + return "GET"sv; + case HttpVerb::kPut: + return "PUT"sv; + case HttpVerb::kPost: + return "POST"sv; + case HttpVerb::kDelete: + return "DELETE"sv; + case HttpVerb::kHead: + return "HEAD"sv; + case HttpVerb::kCopy: + return "COPY"sv; + case HttpVerb::kOptions: + return "OPTIONS"sv; + default: + return "???"sv; + } +} + const char* ReasonStringForHttpResultCode(int HttpCode) { diff --git a/zenhttp/include/zenhttp/httpcommon.h b/zenhttp/include/zenhttp/httpcommon.h index 08f1b47a9..62070061c 100644 --- a/zenhttp/include/zenhttp/httpcommon.h +++ b/zenhttp/include/zenhttp/httpcommon.h @@ -40,6 +40,8 @@ enum class HttpVerb : uint8_t gsl_DEFINE_ENUM_BITMASK_OPERATORS(HttpVerb); +const std::string_view ToString(HttpVerb Verb); + enum class HttpResponseCode { // 1xx - Informational -- cgit v1.2.3 From b527b0a120defc51fc27e8d0a95bc4f689a7e8f4 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 22:06:16 +0200 Subject: Added http_requests logger for (optionally) logging HTTP requests --- zenserver/diag/logging.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 41b140f90..bc7b883b5 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -258,6 +258,18 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) } #endif + // HTTP server request logging + + std::filesystem::path HttpLogPath = GlobalOptions.DataDir / "logs/http.log"; + + auto HttpSink = std::make_shared(zen::WideToUtf8(HttpLogPath.c_str()), + /* max size */ 128 * 1024 * 1024, + /* max files */ 16, + /* rotate on open */ true); + + auto HttpLogger = std::make_shared("http_requests", HttpSink); + spdlog::register_logger(HttpLogger); + // Jupiter - only log HTTP traffic to file auto JupiterLogger = std::make_shared("jupiter", FileSink); -- cgit v1.2.3 From 9e7d8c9f4b47cc6ca9bdcc79282c0144a797110e Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 22:07:11 +0200 Subject: Reimplemented CasLogFile in terms of BasicFile This removes another ATL dependency in favour of something more platform independent --- zenstore/caslog.cpp | 51 +++++++++++++++++--------------------- zenstore/include/zenstore/caslog.h | 10 ++++---- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp index dc6021544..2bac6affd 100644 --- a/zenstore/caslog.cpp +++ b/zenstore/caslog.cpp @@ -17,11 +17,8 @@ #include -#include - -struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- -#include #include +#include ////////////////////////////////////////////////////////////////////////// @@ -48,13 +45,12 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreat { m_RecordSize = RecordSize; - const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; + std::error_code Ec; + m_File.Open(FileName, IsCreate); - HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition); - - if (FAILED(hRes)) + if (Ec) { - throw std::system_error(GetLastError(), std::system_category(), "Failed to open log file '{}'"_format(FileName)); + throw std::system_error(Ec, "Failed to open log file '{}'"_format(FileName)); } uint64_t AppendOffset = 0; @@ -66,7 +62,7 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreat memcpy(Header.Magic, FileHeader::MagicSequence, sizeof Header.Magic); Header.Finalize(); - m_File.Write(&Header, sizeof Header); + m_File.Write(&Header, sizeof Header, 0); AppendOffset = sizeof(FileHeader); @@ -76,7 +72,7 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreat { // Validate header and log contents and prepare for appending/replay FileHeader Header; - m_File.Read(&Header, sizeof Header); + m_File.Read(&Header, sizeof Header, 0); if ((0 != memcmp(Header.Magic, FileHeader::MagicSequence, sizeof Header.Magic)) || (Header.Checksum != Header.ComputeChecksum())) { @@ -84,11 +80,8 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreat throw std::runtime_error("Mangled log header"); } - ULONGLONG Sz; - m_File.GetSize(Sz); - AppendOffset = Sz; - - m_Header = Header; + AppendOffset = m_File.FileSize(); + m_Header = Header; } m_AppendOffset = AppendOffset; @@ -106,8 +99,7 @@ CasLogFile::Close() void CasLogFile::Replay(std::function&& Handler) { - ULONGLONG LogFileSize; - m_File.GetSize(LogFileSize); + uint64_t LogFileSize = m_File.FileSize(); // Ensure we end up on a clean boundary const uint64_t LogBaseOffset = sizeof(FileHeader); @@ -118,18 +110,16 @@ CasLogFile::Replay(std::function&& Handler) return; } + // This should really be streaming the data rather than just + // reading it into memory, though we don't tend to get very + // large logs so it may not matter + const uint64_t LogDataSize = LogEntryCount * m_RecordSize; std::vector ReadBuffer; ReadBuffer.resize(LogDataSize); - m_File.Seek(LogBaseOffset, FILE_BEGIN); - HRESULT hRes = m_File.Read(ReadBuffer.data(), gsl::narrow(LogDataSize)); - - if (FAILED(hRes)) - { - ThrowSystemException(hRes, "Failed to read log file"); - } + m_File.Read(ReadBuffer.data(), LogDataSize, LogBaseOffset); for (int i = 0; i < LogEntryCount; ++i) { @@ -140,11 +130,16 @@ CasLogFile::Replay(std::function&& Handler) void CasLogFile::Append(const void* DataPointer, uint64_t DataSize) { - HRESULT hRes = m_File.Write(DataPointer, gsl::narrow(DataSize)); + ZEN_ASSERT(DataSize == m_RecordSize); + + uint64_t AppendOffset = m_AppendOffset.fetch_add(DataSize); + + std::error_code Ec; + m_File.Write(DataPointer, gsl::narrow(DataSize), AppendOffset, Ec); - if (FAILED(hRes)) + if (Ec) { - ThrowSystemException(hRes, "Failed to write to log file '{}'"_format(PathFromHandle(m_File))); + throw std::system_error(Ec, "Failed to write to log file '{}'"_format(PathFromHandle(m_File.Handle()))); } } diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h index 1fbda0265..00b987383 100644 --- a/zenstore/include/zenstore/caslog.h +++ b/zenstore/include/zenstore/caslog.h @@ -9,9 +9,9 @@ #include #include #include +#include #include -#include #include namespace zen { @@ -47,10 +47,10 @@ private: static_assert(sizeof(FileHeader) == 64); private: - CAtlFile m_File; - FileHeader m_Header; - size_t m_RecordSize = 1; - uint64_t m_AppendOffset = 0; + BasicFile m_File; + FileHeader m_Header; + size_t m_RecordSize = 1; + std::atomic m_AppendOffset = 0; }; template -- cgit v1.2.3 From d84260d180be72b547d04351c91a02e7c75f2b51 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 26 Sep 2021 22:09:13 +0200 Subject: httpsys: Some logging improvements --- zenhttp/httpsys.cpp | 15 ++++++++++++++- zenhttp/httpsys.h | 13 +++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 9c747d17e..2a50388e3 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -575,7 +575,10 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) \/ \/ \/ */ -HttpSysServer::HttpSysServer(unsigned int ThreadCount) : m_ThreadPool(ThreadCount) +HttpSysServer::HttpSysServer(unsigned int ThreadCount) +: m_Log(logging::Get("http")) +, m_RequestLog(logging::Get("http_requests")) +, m_ThreadPool(ThreadCount) { ULONG Result = HttpInitialize(HTTPAPI_VERSION_2, HTTP_INITIALIZE_SERVER, nullptr); @@ -671,6 +674,8 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) else { m_IsOk = true; + + ZEN_INFO("Started http.sys server at '{}'", WideToUtf8(UrlPath)); } } @@ -950,6 +955,14 @@ HttpSysTransaction::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTran return Status::kRequestPending; } + if (m_HttpServer.m_IsRequestLoggingEnabled) + { + if (m_HandlerRequest.has_value()) + { + m_HttpServer.m_RequestLog.info("{} {}", ToString(m_HandlerRequest->RequestVerb()), m_HandlerRequest->RelativeUri()); + } + } + // Transaction done, caller should clean up (delete) this instance return Status::kDone; } diff --git a/zenhttp/httpsys.h b/zenhttp/httpsys.h index 6616817ec..2e51c538f 100644 --- a/zenhttp/httpsys.h +++ b/zenhttp/httpsys.h @@ -20,6 +20,10 @@ # include # include +namespace spdlog { +class logger; +} + namespace zen { /** @@ -56,8 +60,13 @@ private: void UnregisterService(const char* Endpoint, HttpService& Service); private: - bool m_IsOk = false; - bool m_IsHttpInitialized = false; + spdlog::logger& m_Log; + spdlog::logger& m_RequestLog; + spdlog::logger& Log() { return m_Log; } + + bool m_IsOk = false; + bool m_IsHttpInitialized = false; + bool m_IsRequestLoggingEnabled = false; WinIoThreadPool m_ThreadPool; std::wstring m_BaseUri; // http://*:nnnn/ -- cgit v1.2.3 From 92893bf0d029342a7492e485c3f09ed15e002495 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 00:10:14 +0200 Subject: Fixed up some internals for coding conventions --- zencore/timer.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/zencore/timer.cpp b/zencore/timer.cpp index 1e73a7532..88ec89cb7 100644 --- a/zencore/timer.cpp +++ b/zencore/timer.cpp @@ -30,7 +30,7 @@ GetHifreqTimerValue() } uint64_t -internalGetHifreqTimerFrequency() +InternalGetHifreqTimerFrequency() { #if ZEN_PLATFORM_WINDOWS LARGE_INTEGER li; @@ -42,21 +42,23 @@ internalGetHifreqTimerFrequency() #endif } -static uint64_t qpcFreq = internalGetHifreqTimerFrequency(); +static uint64_t QpcFreq = InternalGetHifreqTimerFrequency(); uint64_t GetHifreqTimerFrequency() { - return qpcFreq; + return QpcFreq; } uint64_t GetHifreqTimerFrequencySafe() { - if (!qpcFreq) - qpcFreq = internalGetHifreqTimerFrequency(); + if (!QpcFreq) + { + QpcFreq = InternalGetHifreqTimerFrequency(); + } - return qpcFreq; + return QpcFreq; } ////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From bca8f116ed7a3ab484df62c15e9f49e0167e2b2f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 00:17:45 +0200 Subject: stats: Completed Meter implementation --- zencore/include/zencore/stats.h | 60 ++++++++-- zencore/stats.cpp | 241 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 265 insertions(+), 36 deletions(-) diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h index 7290fd914..0554f620d 100644 --- a/zencore/include/zencore/stats.h +++ b/zencore/include/zencore/stats.h @@ -12,30 +12,44 @@ template class Gauge { public: - Gauge() : m_value{0} {} + Gauge() : m_Value{0} {} + + T Value() const { return m_Value; } + void SetValue(T Value) { m_Value = Value; } private: - T m_value; + std::atomic m_Value; }; +/** Stats counter + * + * A counter is modified by adding or subtracting a value from a current value. + * This would typically be used to track number of requests in flight, number + * of active jobs etc + * + */ class Counter { public: inline void SetValue(uint64_t Value) { m_count = Value; } inline uint64_t Value() const { return m_count; } - inline void Increment(int64_t AddValue) { m_count += AddValue; } - inline void Decrement(int64_t SubValue) { m_count -= SubValue; } - inline void Clear() { m_count = 0; } + inline void Increment(int64_t AddValue) { m_count.fetch_add(AddValue); } + inline void Decrement(int64_t SubValue) { m_count.fetch_sub(SubValue); } + inline void Clear() { m_count.store(0, std::memory_order_release); } private: - std::atomic_uint64_t m_count{0}; + std::atomic m_count{0}; }; -/// -/// Exponential Weighted Moving Average -/// -class EWMA +/** Exponential Weighted Moving Average + + This is very raw, to use as little state as possible. If we + want to use this more broadly in user code we should perhaps + add a more user-friendly wrapper + */ + +class RawEWMA { public: /// @@ -49,16 +63,38 @@ public: double Rate() const; private: - double m_rate = 0; + std::atomic m_Rate = 0; }; /// -/// Tracks rate of events over time (i.e requests/sec) +/// Tracks rate of events over time (i.e requests/sec), using +/// exponential moving averages /// class Meter { public: + Meter(); + ~Meter(); + + double Rate1(); // One-minute rate + double Rate5(); // Five-minute rate + double Rate15(); // Fifteen-minute rate + double MeanRate(); // Mean rate since instantiation of this meter + void Mark(uint64_t Count = 1); // Register one or more events + private: + std::atomic m_TotalCount{0}; // Accumulator counting number of marks since beginning + std::atomic m_PendingCount{0}; // Pending EWMA update accumulator + std::atomic m_StartTick{0}; // Time this was instantiated (for mean) + std::atomic m_LastTick{0}; // Timestamp of last EWMA tick + std::atomic m_Remain{0}; // Tracks the "modulo" of tick time + bool m_IsFirstTick = true; + RawEWMA m_RateM1; + RawEWMA m_RateM5; + RawEWMA m_RateM15; + + void TickIfNecessary(); + void Tick(); }; extern void stats_forcelink(); diff --git a/zencore/stats.cpp b/zencore/stats.cpp index c5187940e..9ae2ddd28 100644 --- a/zencore/stats.cpp +++ b/zencore/stats.cpp @@ -2,6 +2,7 @@ #include "zencore/stats.h" #include +#include "zencore/thread.h" #include "zencore/timer.h" #if ZEN_WITH_TESTS @@ -14,62 +15,254 @@ namespace zen { -static constexpr int kTickInterval = 5; // In seconds -static constexpr double kSecondsPerMinute = 60.0; -static constexpr int kOneMinute = 1; -static constexpr int kFiveMinutes = 5; -static constexpr int kFifteenMinutes = 15; +static constinit int kTickIntervalInSeconds = 5; +static constinit double kSecondsPerMinute = 60.0; +static constinit int kOneMinute = 1; +static constinit int kFiveMinutes = 5; +static constinit int kFifteenMinutes = 15; -static double kM1_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kOneMinute); -static double kM5_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kFiveMinutes); -static double kM15_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kFifteenMinutes); +static const double kM1_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kOneMinute); +static const double kM5_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kFiveMinutes); +static const double kM15_ALPHA = 1.0 - std::exp(-kTickIntervalInSeconds / kSecondsPerMinute / kFifteenMinutes); -static uint64_t CountPerTick = GetHifreqTimerFrequencySafe() * kTickInterval; -static uint64_t CountPerSecond = GetHifreqTimerFrequencySafe(); +static const uint64_t CountPerTick = GetHifreqTimerFrequencySafe() * kTickIntervalInSeconds; +static const uint64_t CountPerSecond = GetHifreqTimerFrequencySafe(); + +////////////////////////////////////////////////////////////////////////// void -EWMA::Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate) +RawEWMA::Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate) { - double InstantRate = double(Count) / Interval; + const double InstantRate = double(Count) / Interval; if (IsInitialUpdate) { - m_rate = InstantRate; + m_Rate.store(InstantRate, std::memory_order_release); } else { - m_rate += Alpha * (InstantRate - m_rate); + m_Rate.fetch_add(Alpha * (InstantRate - m_Rate)); } } double -EWMA::Rate() const +RawEWMA::Rate() const { - return m_rate * CountPerSecond; + return m_Rate.load(std::memory_order_relaxed) * CountPerSecond; +} + +////////////////////////////////////////////////////////////////////////// + +Meter::Meter() : m_StartTick{GetHifreqTimerValue()}, m_LastTick(m_StartTick.load()) +{ +} + +Meter::~Meter() +{ +} + +void +Meter::TickIfNecessary() +{ + uint64_t OldTick = m_LastTick.load(); + const uint64_t NewTick = GetHifreqTimerValue(); + const uint64_t Age = NewTick - OldTick; + + if (Age > CountPerTick) + { + // Ensure only one thread at a time updates the time. This + // works because our tick interval should be sufficiently + // long to ensure two threads don't end up inside this block + + if (m_LastTick.compare_exchange_strong(OldTick, NewTick)) + { + m_Remain.fetch_add(Age); + + do + { + int64_t Remain = m_Remain.load(std::memory_order_relaxed); + + if (Remain < 0) + { + return; + } + + if (m_Remain.compare_exchange_strong(Remain, Remain - CountPerTick)) + { + Tick(); + } + } while (true); + } + } +} + +void +Meter::Tick() +{ + const uint64_t PendingCount = m_PendingCount.exchange(0); + const bool IsFirstTick = m_IsFirstTick; + + if (IsFirstTick) + { + m_IsFirstTick = false; + } + + m_RateM1.Tick(kM1_ALPHA, CountPerTick, PendingCount, IsFirstTick); + m_RateM5.Tick(kM5_ALPHA, CountPerTick, PendingCount, IsFirstTick); + m_RateM15.Tick(kM15_ALPHA, CountPerTick, PendingCount, IsFirstTick); +} + +double +Meter::Rate1() +{ + TickIfNecessary(); + + return m_RateM1.Rate(); +} + +double +Meter::Rate5() +{ + TickIfNecessary(); + + return m_RateM5.Rate(); +} + +double +Meter::Rate15() +{ + TickIfNecessary(); + + return m_RateM15.Rate(); +} + +double +Meter::MeanRate() +{ + const uint64_t Count = m_TotalCount.load(std::memory_order_relaxed); + + if (Count == 0) + { + return 0.0; + } + + const uint64_t Elapsed = GetHifreqTimerValue() - m_StartTick; + + return (double(Count) * GetHifreqTimerFrequency()) / Elapsed; +} + +void +Meter::Mark(uint64_t Count) +{ + TickIfNecessary(); + + m_TotalCount.fetch_add(Count); + m_PendingCount.fetch_add(Count); } ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS -TEST_CASE("Stats") +TEST_CASE("EWMA") { - SUBCASE("Simple") + SUBCASE("Simple_1") + { + RawEWMA Ewma1; + Ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true); + + CHECK(fabs(Ewma1.Rate() - 5) < 0.1); + + for (int i = 0; i < 60; ++i) + { + Ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false); + } + + CHECK(fabs(Ewma1.Rate() - 10) < 0.1); + + for (int i = 0; i < 60; ++i) + { + Ewma1.Tick(kM1_ALPHA, CountPerSecond, 20, false); + } + + CHECK(fabs(Ewma1.Rate() - 20) < 0.1); + } + + SUBCASE("Simple_10") { - EWMA ewma1; - ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true); + RawEWMA Ewma1; + RawEWMA Ewma5; + RawEWMA Ewma15; + Ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true); + Ewma5.Tick(kM5_ALPHA, CountPerSecond, 5, true); + Ewma15.Tick(kM15_ALPHA, CountPerSecond, 5, true); + + CHECK(fabs(Ewma1.Rate() - 5) < 0.1); + CHECK(fabs(Ewma5.Rate() - 5) < 0.1); + CHECK(fabs(Ewma15.Rate() - 5) < 0.1); - CHECK(ewma1.Rate() - 5 < 0.001); + auto Tick1 = [&Ewma1](auto Value) { Ewma1.Tick(kM1_ALPHA, CountPerSecond, Value, false); }; + auto Tick5 = [&Ewma5](auto Value) { Ewma5.Tick(kM5_ALPHA, CountPerSecond, Value, false); }; + auto Tick15 = [&Ewma15](auto Value) { Ewma15.Tick(kM15_ALPHA, CountPerSecond, Value, false); }; for (int i = 0; i < 60; ++i) - ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false); + { + Tick1(10); + Tick5(10); + Tick15(10); + } + + CHECK(fabs(Ewma1.Rate() - 10) < 0.1); + + for (int i = 0; i < 5 * 60; ++i) + { + Tick1(20); + Tick5(20); + Tick15(20); + } - CHECK(ewma1.Rate() - 10 < 0.001); + CHECK(fabs(Ewma1.Rate() - 20) < 0.1); + CHECK(fabs(Ewma5.Rate() - 20) < 0.1); - ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false); + for (int i = 0; i < 16 * 60; ++i) + { + Tick1(100); + Tick5(100); + Tick15(100); + } + + CHECK(fabs(Ewma1.Rate() - 100) < 0.1); + CHECK(fabs(Ewma5.Rate() - 100) < 0.1); + CHECK(fabs(Ewma15.Rate() - 100) < 0.5); } } +# if 0 // This is not really a unit test, but mildly useful to exercise some code +TEST_CASE("Meter") +{ + Meter Meter1; + Meter1.Mark(1); + Sleep(1000); + Meter1.Mark(1); + Sleep(1000); + Meter1.Mark(1); + Sleep(1000); + Meter1.Mark(1); + Sleep(1000); + Meter1.Mark(1); + Sleep(1000); + Meter1.Mark(1); + Sleep(1000); + Meter1.Mark(1); + Sleep(1000); + Meter1.Mark(1); + Sleep(1000); + Meter1.Mark(1); + Sleep(1000); + [[maybe_unused]] double Rate = Meter1.MeanRate(); +} +# endif + void stats_forcelink() { -- cgit v1.2.3 From b34490abe0f19f22499fb9dafb9b7431ca5ecc95 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 27 Sep 2021 11:02:38 +0200 Subject: Ported CbValue from Unreal to Zen (#10) CompactBinary: Ported CbValue changes from UE5 --- zencore/compactbinary.cpp | 172 ++++++++-------- zencore/include/zencore/compactbinary.h | 80 ++++++-- zencore/include/zencore/compactbinaryvalue.h | 290 +++++++++++++++++++++++++++ zencore/include/zencore/memory.h | 6 + 4 files changed, 439 insertions(+), 109 deletions(-) create mode 100644 zencore/include/zencore/compactbinaryvalue.h diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index f4908aa9a..a68096c36 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -3,6 +3,7 @@ #include "zencore/compactbinary.h" #include +#include #include #include #include @@ -66,21 +67,8 @@ TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano) ////////////////////////////////////////////////////////////////////////// namespace CompactBinaryPrivate { - static constexpr const uint8_t GEmptyObjectPayload[] = {uint8_t(CbFieldType::Object), 0x00}; static constexpr const uint8_t GEmptyArrayPayload[] = {uint8_t(CbFieldType::Array), 0x01, 0x00}; - - template - static constexpr inline T ReadUnaligned(const void* const Memory) - { -#if PLATFORM_SUPPORTS_UNALIGNED_LOADS - return *static_cast(Memory); -#else - T Value; - memcpy(&Value, Memory, sizeof(Value)); - return Value; -#endif - } } // namespace CompactBinaryPrivate ////////////////////////////////////////////////////////////////////////// @@ -151,14 +139,10 @@ CbFieldView::AsArrayView() MemoryView CbFieldView::AsBinaryView(const MemoryView Default) { - if (CbFieldTypeOps::IsBinary(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsBinary(Accessor.GetType())) { - const uint8_t* const PayloadBytes = static_cast(Payload); - uint32_t ValueSizeByteCount; - const uint64_t ValueSize = ReadVarUInt(PayloadBytes, ValueSizeByteCount); - Error = CbFieldError::None; - return MemoryView(PayloadBytes + ValueSizeByteCount, ValueSize); + return Accessor.AsBinary(); } else { @@ -170,20 +154,25 @@ CbFieldView::AsBinaryView(const MemoryView Default) std::string_view CbFieldView::AsString(const std::string_view Default) { - if (CbFieldTypeOps::IsString(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsString(Accessor.GetType())) { - const char* const PayloadChars = static_cast(Payload); - uint32_t ValueSizeByteCount; - const uint64_t ValueSize = ReadVarUInt(PayloadChars, ValueSizeByteCount); - - if (ValueSize >= (uint64_t(1) << 31)) - { - Error = CbFieldError::RangeError; - return Default; - } + Error = CbFieldError::None; + return Accessor.AsString(); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} +std::u8string_view +CbFieldView::AsU8String(const std::u8string_view Default) +{ + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsString(Accessor.GetType())) + { Error = CbFieldError::None; - return std::string_view(PayloadChars + ValueSizeByteCount, ValueSize); + return Accessor.AsU8String(); } else { @@ -193,23 +182,11 @@ CbFieldView::AsString(const std::string_view Default) } uint64_t -CbFieldView::AsInteger(const uint64_t Default, const IntegerParams Params) +CbFieldView::AsInteger(const uint64_t Default, const CompactBinaryPrivate::IntegerParams Params) { - if (CbFieldTypeOps::IsInteger(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsInteger(Accessor.GetType())) { - // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero. - const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1); - const uint64_t IsNegative = uint8_t(Type) & 1; - - uint32_t MagnitudeByteCount; - const uint64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount); - const uint64_t Value = Magnitude ^ -int64_t(IsNegative); - - const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned); - Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError; - - const uint64_t UseValueMask = -int64_t(IsInRange); - return (Value & UseValueMask) | (Default & ~UseValueMask); + return Accessor.AsInteger(Params, &Error, Default); } else { @@ -221,25 +198,24 @@ CbFieldView::AsInteger(const uint64_t Default, const IntegerParams Params) float CbFieldView::AsFloat(const float Default) { - switch (CbFieldTypeOps::GetType(Type)) + switch (CbValue Accessor = GetValue(); Accessor.GetType()) { case CbFieldType::IntegerPositive: case CbFieldType::IntegerNegative: { - const uint64_t IsNegative = uint8_t(Type) & 1; + const uint64_t IsNegative = uint8_t(Accessor.GetType()) & 1; constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*FLT_MANT_DIG*/ 24) - 1); uint32_t MagnitudeByteCount; - const int64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount) + IsNegative; + const int64_t Magnitude = ReadVarUInt(Accessor.GetData(), MagnitudeByteCount) + IsNegative; const uint64_t IsInRange = !(Magnitude & OutOfRangeMask); Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError; return IsInRange ? float(IsNegative ? -Magnitude : Magnitude) : Default; } case CbFieldType::Float32: { - Error = CbFieldError::None; - const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Payload)); - return reinterpret_cast(Value); + Error = CbFieldError::None; + return Accessor.AsFloat32(); } case CbFieldType::Float64: Error = CbFieldError::RangeError; @@ -253,31 +229,29 @@ CbFieldView::AsFloat(const float Default) double CbFieldView::AsDouble(const double Default) { - switch (CbFieldTypeOps::GetType(Type)) + switch (CbValue Accessor = GetValue(); Accessor.GetType()) { case CbFieldType::IntegerPositive: case CbFieldType::IntegerNegative: { - const uint64_t IsNegative = uint8_t(Type) & 1; + const uint64_t IsNegative = uint8_t(Accessor.GetType()) & 1; constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*DBL_MANT_DIG*/ 53) - 1); uint32_t MagnitudeByteCount; - const int64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount) + IsNegative; + const int64_t Magnitude = ReadVarUInt(Accessor.GetData(), MagnitudeByteCount) + IsNegative; const uint64_t IsInRange = !(Magnitude & OutOfRangeMask); Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError; return IsInRange ? double(IsNegative ? -Magnitude : Magnitude) : Default; } case CbFieldType::Float32: { - Error = CbFieldError::None; - const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Payload)); - return reinterpret_cast(Value); + Error = CbFieldError::None; + return Accessor.AsFloat32(); } case CbFieldType::Float64: { - Error = CbFieldError::None; - const uint64_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Payload)); - return reinterpret_cast(Value); + Error = CbFieldError::None; + return Accessor.AsFloat64(); } default: Error = CbFieldError::TypeError; @@ -288,19 +262,19 @@ CbFieldView::AsDouble(const double Default) bool CbFieldView::AsBool(const bool bDefault) { - const CbFieldType LocalType = Type; - const bool bIsBool = CbFieldTypeOps::IsBool(LocalType); - Error = bIsBool ? CbFieldError::None : CbFieldError::TypeError; - return (uint8_t(bIsBool) & uint8_t(LocalType) & 1) | ((!bIsBool) & bDefault); + CbValue Accessor = GetValue(); + const bool IsBool = CbFieldTypeOps::IsBool(Accessor.GetType()); + Error = IsBool ? CbFieldError::None : CbFieldError::TypeError; + return (uint8_t(IsBool) & Accessor.AsBool()) | ((!IsBool) & bDefault); } IoHash CbFieldView::AsObjectAttachment(const IoHash& Default) { - if (CbFieldTypeOps::IsObjectAttachment(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsObjectAttachment(Accessor.GetType())) { Error = CbFieldError::None; - return IoHash::MakeFrom(Payload); + return Accessor.AsObjectAttachment(); } else { @@ -312,10 +286,10 @@ CbFieldView::AsObjectAttachment(const IoHash& Default) IoHash CbFieldView::AsBinaryAttachment(const IoHash& Default) { - if (CbFieldTypeOps::IsBinaryAttachment(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsBinaryAttachment(Accessor.GetType())) { Error = CbFieldError::None; - return IoHash::MakeFrom(Payload); + return Accessor.AsBinaryAttachment(); } else { @@ -327,10 +301,10 @@ CbFieldView::AsBinaryAttachment(const IoHash& Default) IoHash CbFieldView::AsAttachment(const IoHash& Default) { - if (CbFieldTypeOps::IsAttachment(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsAttachment(Accessor.GetType())) { Error = CbFieldError::None; - return IoHash::MakeFrom(Payload); + return Accessor.AsAttachment(); } else { @@ -342,10 +316,10 @@ CbFieldView::AsAttachment(const IoHash& Default) IoHash CbFieldView::AsHash(const IoHash& Default) { - if (CbFieldTypeOps::IsHash(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsHash(Accessor.GetType())) { Error = CbFieldError::None; - return IoHash::MakeFrom(Payload); + return Accessor.AsHash(); } else { @@ -363,16 +337,10 @@ CbFieldView::AsUuid() Guid CbFieldView::AsUuid(const Guid& Default) { - if (CbFieldTypeOps::IsUuid(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsUuid(Accessor.GetType())) { Error = CbFieldError::None; - Guid Value; - memcpy(&Value, Payload, sizeof(Guid)); - Value.A = FromNetworkOrder(Value.A); - Value.B = FromNetworkOrder(Value.B); - Value.C = FromNetworkOrder(Value.C); - Value.D = FromNetworkOrder(Value.D); - return Value; + return Accessor.AsUuid(); } else { @@ -390,12 +358,40 @@ CbFieldView::AsObjectId() Oid CbFieldView::AsObjectId(const Oid& Default) { - if (CbFieldTypeOps::IsObjectId(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsObjectId(Accessor.GetType())) + { + Error = CbFieldError::None; + return Accessor.AsObjectId(); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +CbCustomById +CbFieldView::AsCustomById(CbCustomById Default) +{ + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsCustomById(Accessor.GetType())) + { + Error = CbFieldError::None; + return Accessor.AsCustomById(); + } + else + { + Error = CbFieldError::TypeError; + return Default; + } +} + +CbCustomByName +CbFieldView::AsCustomByName(CbCustomByName Default) +{ + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsCustomByName(Accessor.GetType())) { Error = CbFieldError::None; - Oid Value; - memcpy(&Value, Payload, sizeof(Oid)); - return Value; + return Accessor.AsCustomByName(); } else { @@ -407,10 +403,10 @@ CbFieldView::AsObjectId(const Oid& Default) int64_t CbFieldView::AsDateTimeTicks(const int64_t Default) { - if (CbFieldTypeOps::IsDateTime(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsDateTime(Accessor.GetType())) { Error = CbFieldError::None; - return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Payload)); + return Accessor.AsDateTimeTicks(); } else { @@ -434,10 +430,10 @@ CbFieldView::AsDateTime(DateTime Default) int64_t CbFieldView::AsTimeSpanTicks(const int64_t Default) { - if (CbFieldTypeOps::IsTimeSpan(Type)) + if (CbValue Accessor = GetValue(); CbFieldTypeOps::IsTimeSpan(Accessor.GetType())) { Error = CbFieldError::None; - return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Payload)); + return Accessor.AsTimeSpanTicks(); } else { diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index 4fce129ea..60e9ec7aa 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -30,6 +30,7 @@ class CbArrayView; class BinaryReader; class BinaryWriter; class CompressedBuffer; +class CbValue; class DateTime { @@ -318,6 +319,9 @@ public: static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; } static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; } + static constexpr inline bool IsCustomById(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomById; } + static constexpr inline bool IsCustomByName(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomByName; } + static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; } static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; } @@ -365,6 +369,46 @@ public: virtual void VisitTimeSpan(TimeSpan Value) = 0; }; +/** A custom compact binary field type with an integer identifier. */ +struct CbCustomById +{ + /** An identifier for the sub-type of the field. */ + uint64_t Id = 0; + /** A view of the value. Lifetime is tied to the field that the value is associated with. */ + MemoryView Data; +}; + +/** A custom compact binary field type with a string identifier. */ +struct CbCustomByName +{ + /** An identifier for the sub-type of the field. Lifetime is tied to the field that the name is associated with. */ + std::u8string_view Name; + /** A view of the value. Lifetime is tied to the field that the value is associated with. */ + MemoryView Data; +}; + +namespace CompactBinaryPrivate { + /** Parameters for converting to an integer. */ + struct IntegerParams + { + /** Whether the output type has a sign bit. */ + uint32_t IsSigned : 1; + /** Bits of magnitude. (7 for int8) */ + uint32_t MagnitudeBits : 31; + }; + + /** Make integer params for the given integer type. */ + template + static constexpr inline IntegerParams MakeIntegerParams() + { + IntegerParams Params; + Params.IsSigned = IntType(-1) < IntType(0); + Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned; + return Params; + } + +} // namespace CompactBinaryPrivate + /** * An atom of data in the compact binary format. * @@ -393,13 +437,20 @@ public: ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType); + /** Construct a field from a value, without access to the name. */ + inline explicit CbFieldView(const CbValue& Value); + /** Returns the name of the field if it has a name, otherwise an empty view. */ constexpr inline std::string_view GetName() const { return std::string_view(static_cast(Payload) - NameLen, NameLen); } + /** Returns the value for unchecked access. Prefer the typed accessors below. */ + inline CbValue GetValue() const; + ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView()); ZENCORE_API CbObjectView AsObjectView(); ZENCORE_API CbArrayView AsArrayView(); ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view()); + ZENCORE_API std::u8string_view AsU8String(std::u8string_view Default = std::u8string_view()); ZENCORE_API void IterateAttachments(std::function Visitor) const; @@ -448,6 +499,11 @@ public: /** Access the field as a OID. Returns the provided default on error. */ ZENCORE_API Oid AsObjectId(const Oid& Default); + /** Access the field as a custom sub-type with an integer identifier. Returns the provided default on error. */ + ZENCORE_API CbCustomById AsCustomById(CbCustomById Default); + /** Access the field as a custom sub-type with a string identifier. Returns the provided default on error. */ + ZENCORE_API CbCustomByName AsCustomByName(CbCustomByName Default); + /** Access the field as a date/time tick count. Returns the provided default on error. */ ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0); @@ -590,25 +646,6 @@ protected: } private: - /** Parameters for converting to an integer. */ - struct IntegerParams - { - /** Whether the output type has a sign bit. */ - uint32_t IsSigned : 1; - /** Bits of magnitude. (7 for int8) */ - uint32_t MagnitudeBits : 31; - }; - - /** Make integer params for the given integer type. */ - template - static constexpr inline IntegerParams MakeIntegerParams() - { - IntegerParams Params; - Params.IsSigned = IntType(-1) < IntType(0); - Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned; - return Params; - } - /** * Access the field as the given integer type. * @@ -617,11 +654,12 @@ private: template inline IntType AsInteger(IntType Default) { - return IntType(AsInteger(uint64_t(Default), MakeIntegerParams())); + return IntType(AsInteger(uint64_t(Default), CompactBinaryPrivate::MakeIntegerParams())); } - ZENCORE_API uint64_t AsInteger(uint64_t Default, IntegerParams Params); + ZENCORE_API uint64_t AsInteger(uint64_t Default, CompactBinaryPrivate::IntegerParams Params); +private: /** The field type, with the transient HasFieldType flag if the field contains its type. */ CbFieldType Type = CbFieldType::None; /** The error (if any) that occurred on the last field access. */ diff --git a/zencore/include/zencore/compactbinaryvalue.h b/zencore/include/zencore/compactbinaryvalue.h new file mode 100644 index 000000000..5795ef957 --- /dev/null +++ b/zencore/include/zencore/compactbinaryvalue.h @@ -0,0 +1,290 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include + +namespace zen { + +namespace CompactBinaryPrivate { + + template + static constexpr inline T ReadUnaligned(const void* const Memory) + { +#if PLATFORM_SUPPORTS_UNALIGNED_LOADS + return *static_cast(Memory); +#else + T Value; + memcpy(&Value, Memory, sizeof(Value)); + return Value; +#endif + } +} // namespace CompactBinaryPrivate +/** + * A type that provides unchecked access to compact binary values. + * + * The main purpose of the type is to efficiently switch on field type. For every other use case, + * prefer to use the field, array, and object types directly. The accessors here do not check the + * type before reading the value, which means they can read out of bounds even on a valid compact + * binary value if the wrong accessor is used. + */ +class CbValue +{ +public: + CbValue(CbFieldType Type, const void* Value); + + CbObjectView AsObjectView() const; + CbArrayView AsArrayView() const; + + MemoryView AsBinary() const; + + /** Access as a string. Checks for range errors and uses the default if OutError is not null. */ + std::string_view AsString(CbFieldError* OutError = nullptr, std::string_view Default = std::string_view()) const; + + /** Access as a string as UTF8. Checks for range errors and uses the default if OutError is not null. */ + std::u8string_view AsU8String(CbFieldError* OutError = nullptr, std::u8string_view Default = std::u8string_view()) const; + + /** + * Access as an integer, with both positive and negative values returned as unsigned. + * + * Checks for range errors and uses the default if OutError is not null. + */ + uint64_t AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError = nullptr, uint64_t Default = 0) const; + + uint64_t AsIntegerPositive() const; + int64_t AsIntegerNegative() const; + + float AsFloat32() const; + double AsFloat64() const; + + bool AsBool() const; + + inline IoHash AsObjectAttachment() const { return AsHash(); } + inline IoHash AsBinaryAttachment() const { return AsHash(); } + inline IoHash AsAttachment() const { return AsHash(); } + + IoHash AsHash() const; + Guid AsUuid() const; + + int64_t AsDateTimeTicks() const; + int64_t AsTimeSpanTicks() const; + + Oid AsObjectId() const; + + CbCustomById AsCustomById() const; + CbCustomByName AsCustomByName() const; + + inline CbFieldType GetType() const { return Type; } + inline const void* GetData() const { return Data; } + +private: + const void* Data; + CbFieldType Type; +}; + +inline CbFieldView::CbFieldView(const CbValue& InValue) : Type(InValue.GetType()), Payload(InValue.GetData()) +{ +} + +inline CbValue +CbFieldView::GetValue() const +{ + return CbValue(CbFieldTypeOps::GetType(Type), Payload); +} + +inline CbValue::CbValue(CbFieldType InType, const void* InValue) : Data(InValue), Type(InType) +{ +} + +inline CbObjectView +CbValue::AsObjectView() const +{ + return CbObjectView(*this); +} + +inline CbArrayView +CbValue::AsArrayView() const +{ + return CbArrayView(*this); +} + +inline MemoryView +CbValue::AsBinary() const +{ + const uint8_t* const Bytes = static_cast(Data); + uint32_t ValueSizeByteCount; + const uint64_t ValueSize = ReadVarUInt(Bytes, ValueSizeByteCount); + return MakeMemoryView(Bytes + ValueSizeByteCount, ValueSize); +} + +inline std::string_view +CbValue::AsString(CbFieldError* OutError, std::string_view Default) const +{ + const char* const Chars = static_cast(Data); + uint32_t ValueSizeByteCount; + const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); + + if (OutError) + { + if (ValueSize >= (uint64_t(1) << 31)) + { + *OutError = CbFieldError::RangeError; + return Default; + } + *OutError = CbFieldError::None; + } + + return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); +} + +inline std::u8string_view +CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const +{ + const char8_t* const Chars = static_cast(Data); + uint32_t ValueSizeByteCount; + const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); + + if (OutError) + { + if (ValueSize >= (uint64_t(1) << 31)) + { + *OutError = CbFieldError::RangeError; + return Default; + } + *OutError = CbFieldError::None; + } + + return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); +} + +inline uint64_t +CbValue::AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError, uint64_t Default) const +{ + // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero. + const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1); + const uint64_t IsNegative = uint8_t(Type) & 1; + + uint32_t MagnitudeByteCount; + const uint64_t Magnitude = ReadVarUInt(Data, MagnitudeByteCount); + const uint64_t Value = Magnitude ^ -int64_t(IsNegative); + + if (OutError) + { + const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned); + *OutError = IsInRange ? CbFieldError::None : CbFieldError::RangeError; + + const uint64_t UseValueMask = -int64_t(IsInRange); + return (Value & UseValueMask) | (Default & ~UseValueMask); + } + + return Value; +} + +inline uint64_t +CbValue::AsIntegerPositive() const +{ + uint32_t MagnitudeByteCount; + return ReadVarUInt(Data, MagnitudeByteCount); +} + +inline int64_t +CbValue::AsIntegerNegative() const +{ + uint32_t MagnitudeByteCount; + return int64_t(ReadVarUInt(Data, MagnitudeByteCount)) ^ -int64_t(1); +} + +inline float +CbValue::AsFloat32() const +{ + const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Data)); + return reinterpret_cast(Value); +} + +inline double +CbValue::AsFloat64() const +{ + const uint64_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Data)); + return reinterpret_cast(Value); +} + +inline bool +CbValue::AsBool() const +{ + return uint8_t(Type) & 1; +} + +inline IoHash +CbValue::AsHash() const +{ + return IoHash::MakeFrom(Data); +} + +inline Guid +CbValue::AsUuid() const +{ + Guid Value; + memcpy(&Value, Data, sizeof(Guid)); + Value.A = FromNetworkOrder(Value.A); + Value.B = FromNetworkOrder(Value.B); + Value.C = FromNetworkOrder(Value.C); + Value.D = FromNetworkOrder(Value.D); + return Value; +} + +inline int64_t +CbValue::AsDateTimeTicks() const +{ + return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Data)); +} + +inline int64_t +CbValue::AsTimeSpanTicks() const +{ + return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned(Data)); +} + +inline Oid +CbValue::AsObjectId() const +{ + return Oid::FromMemory(Data); +} + +inline CbCustomById +CbValue::AsCustomById() const +{ + const uint8_t* Bytes = static_cast(Data); + uint32_t DataSizeByteCount; + const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount); + Bytes += DataSizeByteCount; + + CbCustomById Value; + uint32_t TypeIdByteCount; + Value.Id = ReadVarUInt(Bytes, TypeIdByteCount); + Value.Data = MakeMemoryView(Bytes + TypeIdByteCount, DataSize - TypeIdByteCount); + return Value; +} + +inline CbCustomByName +CbValue::AsCustomByName() const +{ + const uint8_t* Bytes = static_cast(Data); + uint32_t DataSizeByteCount; + const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount); + Bytes += DataSizeByteCount; + + uint32_t TypeNameLenByteCount; + const uint64_t TypeNameLen = ReadVarUInt(Bytes, TypeNameLenByteCount); + Bytes += TypeNameLenByteCount; + + CbCustomByName Value; + Value.Name = std::u8string_view(reinterpret_cast(Bytes), static_cast(TypeNameLen)); + Value.Data = MakeMemoryView(Bytes + TypeNameLen, DataSize - TypeNameLen - TypeNameLenByteCount); + return Value; +} + +} // namespace zen diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h index 3d4db1081..aba391c85 100644 --- a/zencore/include/zencore/memory.h +++ b/zencore/include/zencore/memory.h @@ -354,6 +354,12 @@ MakeMemoryView(const void* Data, const void* DataEnd) return MemoryView(Data, DataEnd); } +[[nodiscard]] inline MemoryView +MakeMemoryView(const void* Data, uint64_t Size) +{ + return MemoryView(Data, reinterpret_cast(Data) + Size); +} + /** * Make a non-owning mutable view of the memory of the initializer list. * -- cgit v1.2.3 From 895b22378d310c4affb782267ae1cbb3bc725c36 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 27 Sep 2021 11:17:18 +0200 Subject: Compact binary to JSON (#12) CompactBinary: Support for converting CbObject to JSON --- zencore/base64.cpp | 105 +++++++ zencore/compactbinary.cpp | 538 ++++++++++++++++++++++++++++++++ zencore/include/zencore/base64.h | 17 + zencore/include/zencore/compactbinary.h | 55 +++- zencore/include/zencore/string.h | 319 +++++++++++++++++++ zencore/include/zencore/zencore.h | 2 + zencore/zencore.vcxproj | 2 + zencore/zencore.vcxproj.filters | 2 + zenhttp/httpserver.cpp | 14 +- 9 files changed, 1049 insertions(+), 5 deletions(-) create mode 100644 zencore/base64.cpp create mode 100644 zencore/include/zencore/base64.h diff --git a/zencore/base64.cpp b/zencore/base64.cpp new file mode 100644 index 000000000..9a6ea3b75 --- /dev/null +++ b/zencore/base64.cpp @@ -0,0 +1,105 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +namespace zen { + +/** The table used to encode a 6 bit value as an ascii character */ +static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; + +/** The table used to convert an ascii character into a 6 bit value */ +static const uint8_t DecodingAlphabet[256] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x3E, 0xFF, 0xFF, 0xFF, 0x3F, // 0x20-0x2f + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x30-0x3f + 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, // 0x40-0x4f + 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x50-0x5f + 0xFF, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, // 0x60-0x6f + 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x70-0x7f + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x80-0x8f + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x90-0x9f + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xa0-0xaf + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xb0-0xbf + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xc0-0xcf + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xd0-0xdf + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff +}; + +template +uint32_t +Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest) +{ + CharType* EncodedBytes = Dest; + + // Loop through the buffer converting 3 bytes of binary data at a time + while (Length >= 3) + { + uint8_t A = *Source++; + uint8_t B = *Source++; + uint8_t C = *Source++; + Length -= 3; + + // The algorithm takes 24 bits of data (3 bytes) and breaks it into 4 6bit chunks represented as ascii + uint32_t ByteTriplet = A << 16 | B << 8 | C; + + // Use the 6bit block to find the representation ascii character for it + EncodedBytes[3] = EncodingAlphabet[ByteTriplet & 0x3F]; + ByteTriplet >>= 6; + EncodedBytes[2] = EncodingAlphabet[ByteTriplet & 0x3F]; + ByteTriplet >>= 6; + EncodedBytes[1] = EncodingAlphabet[ByteTriplet & 0x3F]; + ByteTriplet >>= 6; + EncodedBytes[0] = EncodingAlphabet[ByteTriplet & 0x3F]; + + // Now we can append this buffer to our destination string + EncodedBytes += 4; + } + + // Since this algorithm operates on blocks, we may need to pad the last chunks + if (Length > 0) + { + uint8_t A = *Source++; + uint8_t B = 0; + uint8_t C = 0; + // Grab the second character if it is a 2 uint8_t finish + if (Length == 2) + { + B = *Source; + } + uint32_t ByteTriplet = A << 16 | B << 8 | C; + // Pad with = to make a 4 uint8_t chunk + EncodedBytes[3] = '='; + ByteTriplet >>= 6; + // If there's only one 1 uint8_t left in the source, then you need 2 pad chars + if (Length == 1) + { + EncodedBytes[2] = '='; + } + else + { + EncodedBytes[2] = EncodingAlphabet[ByteTriplet & 0x3F]; + } + // Now encode the remaining bits the same way + ByteTriplet >>= 6; + EncodedBytes[1] = EncodingAlphabet[ByteTriplet & 0x3F]; + ByteTriplet >>= 6; + EncodedBytes[0] = EncodingAlphabet[ByteTriplet & 0x3F]; + + EncodedBytes += 4; + } + + // Add a null terminator + *EncodedBytes = 0; + + return uint32_t(EncodedBytes - Dest); +} + +template ZENCORE_API uint32_t Base64::Encode(const uint8_t* Source, uint32_t Length, char* Dest); +template ZENCORE_API uint32_t Base64::Encode(const uint8_t* Source, uint32_t Length, wchar_t* Dest); + +} // namespace zen diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index a68096c36..3b6d33e41 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -2,19 +2,35 @@ #include "zencore/compactbinary.h" +#include #include #include #include #include +#include #include +#include #include +#include +#include #include +#if ZEN_WITH_TESTS +# include +# include +#endif + namespace zen { const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}; +double +GetJulianDay(uint64_t Ticks) +{ + return (double)(1721425.5 + Ticks / TimeSpan::TicksPerDay); +} + bool IsLeapYear(int Year) { @@ -64,6 +80,249 @@ TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano) Ticks = TotalTicks; } +std::string +TimeSpan::ToString(const char* Format) const +{ + using namespace fmt::literals; + + StringBuilder<128> Result; + + Result.Append((Ticks < 0) ? '-' : '+'); + + while (*Format != '\0') + { + if ((*Format == '%') && (*++Format != '\0')) + { + switch (*Format) + { + case 'd': + Result.Append("%i"_format(GetDays())); + break; + case 'D': + Result.Append("%08i"_format(GetDays())); + break; + case 'h': + Result.Append("%02i"_format(GetHours())); + break; + case 'm': + Result.Append("%02i"_format(GetMinutes())); + break; + case 's': + Result.Append("%02i"_format(GetSeconds())); + break; + case 'f': + Result.Append("%03i"_format(GetFractionMilli())); + break; + case 'u': + Result.Append("%06i"_format(GetFractionMicro())); + break; + case 't': + Result.Append("%07i"_format(GetFractionTicks())); + break; + case 'n': + Result.Append("%09i"_format(GetFractionNano())); + break; + default: + Result.Append(*Format); + } + } + else + { + Result.Append(*Format); + } + + ++Format; + } + + return Result.ToString(); +} + +std::string +TimeSpan::ToString() const +{ + if (GetDays() == 0) + { + return ToString("%h:%m:%s.%f"); + } + + return ToString("%d.%h:%m:%s.%f"); +} + +int +DateTime::GetYear() const +{ + int Year, Month, Day; + GetDate(Year, Month, Day); + + return Year; +} + +int +DateTime::GetMonth() const +{ + int Year, Month, Day; + GetDate(Year, Month, Day); + + return Month; +} + +int +DateTime::GetDay() const +{ + int Year, Month, Day; + GetDate(Year, Month, Day); + + return Day; +} + +int +DateTime::GetHour() const +{ + return (int)((Ticks / TimeSpan::TicksPerHour) % 24); +} + +int +DateTime::GetHour12() const +{ + int Hour = GetHour(); + + if (Hour < 1) + { + return 12; + } + + if (Hour > 12) + { + return (Hour - 12); + } + + return Hour; +} + +int +DateTime::GetMinute() const +{ + return (int)((Ticks / TimeSpan::TicksPerMinute) % 60); +} + +int +DateTime::GetSecond() const +{ + return (int)((Ticks / TimeSpan::TicksPerSecond) % 60); +} + +int +DateTime::GetMillisecond() const +{ + return (int)((Ticks / TimeSpan::TicksPerMillisecond) % 1000); +} + +void +DateTime::GetDate(int& Year, int& Month, int& Day) const +{ + // Based on FORTRAN code in: + // Fliegel, H. F. and van Flandern, T. C., + // Communications of the ACM, Vol. 11, No. 10 (October 1968). + + int i, j, k, l, n; + + l = int(GetJulianDay(Ticks) + 0.5) + 68569; + n = 4 * l / 146097; + l = l - (146097 * n + 3) / 4; + i = 4000 * (l + 1) / 1461001; + l = l - 1461 * i / 4 + 31; + j = 80 * l / 2447; + k = l - 2447 * j / 80; + l = j / 11; + j = j + 2 - 12 * l; + i = 100 * (n - 49) + i + l; + + Year = i; + Month = j; + Day = k; +} + +std::string +DateTime::ToString(const char* Format) const +{ + using namespace fmt::literals; + + StringBuilder<32> Result; + int Year, Month, Day; + + GetDate(Year, Month, Day); + + if (Format != nullptr) + { + while (*Format != '\0') + { + if ((*Format == '%') && (*(++Format) != '\0')) + { + switch (*Format) + { + // case 'a': Result.Append(IsMorning() ? TEXT("am") : TEXT("pm")); break; + // case 'A': Result.Append(IsMorning() ? TEXT("AM") : TEXT("PM")); break; + case 'd': + Result.Append("%02i"_format(Day)); + break; + // case 'D': Result.Appendf(TEXT("%03i"), GetDayOfYear()); break; + case 'm': + Result.Append("%02i"_format(Month)); + break; + case 'y': + Result.Append("%02i"_format(Year % 100)); + break; + case 'Y': + Result.Append("%04i"_format(Year % 100)); + break; + case 'h': + Result.Append("%02i"_format(GetHour12())); + break; + case 'H': + Result.Append("%02i"_format(GetHour())); + break; + case 'M': + Result.Append("%02i"_format(GetMinute())); + break; + case 'S': + Result.Append("%02i"_format(GetSecond())); + break; + case 's': + Result.Append("%03i"_format(GetMillisecond())); + break; + default: + Result.Append(*Format); + } + } + else + { + Result.Append(*Format); + } + + // move to the next one + Format++; + } + } + + return Result.ToString(); +} + +std::string +DateTime::ToIso8601() const +{ + return ToString("%Y-%m-%dT%H:%M:%S.%sZ"); +} + +StringBuilderBase& +Guid::ToString(StringBuilderBase& Sb) const +{ + char Buf[128]; + snprintf(Buf, sizeof Buf, "%08x-%04x-%04x-%04x-%04x%08x", A, B >> 16, B & 0xFFFF, C >> 16, C & 0xFFFF, D); + Sb << Buf; + + return Sb; +} + ////////////////////////////////////////////////////////////////////////// namespace CompactBinaryPrivate { @@ -1142,6 +1401,259 @@ SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object) ////////////////////////////////////////////////////////////////////////// +class CbJsonWriter +{ +public: + explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } + + void WriteField(CbFieldView Field) + { + using namespace fmt::literals; + using namespace std::literals; + + WriteOptionalComma(); + WriteOptionalNewLine(); + + if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) + { + AppendQuotedString(Name); + Builder << ": "sv; + } + + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::Null: + Builder << "null"sv; + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + { + Builder << '{'; + NewLineAndIndent << '\t'; + NeedsNewLine = true; + for (CbFieldView It : Field) + { + WriteField(It); + } + NewLineAndIndent.RemoveSuffix(1); + if (NeedsComma) + { + WriteOptionalNewLine(); + } + Builder << '}'; + } + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + { + Builder << '['; + NewLineAndIndent << '\t'; + NeedsNewLine = true; + for (CbFieldView It : Field) + { + WriteField(It); + } + NewLineAndIndent.RemoveSuffix(1); + if (NeedsComma) + { + WriteOptionalNewLine(); + } + Builder << ']'; + } + break; + case CbFieldType::Binary: + AppendBase64String(Accessor.AsBinary()); + break; + case CbFieldType::String: + AppendQuotedString(Accessor.AsU8String()); + break; + case CbFieldType::IntegerPositive: + Builder << Accessor.AsIntegerPositive(); + break; + case CbFieldType::IntegerNegative: + Builder << Accessor.AsIntegerNegative(); + break; + case CbFieldType::Float32: + Builder.Append("%.9g"_format(Accessor.AsFloat32())); + break; + case CbFieldType::Float64: + Builder.Append("%.17g"_format(Accessor.AsFloat64())); + break; + case CbFieldType::BoolFalse: + Builder << "false"sv; + break; + case CbFieldType::BoolTrue: + Builder << "true"sv; + break; + case CbFieldType::ObjectAttachment: + case CbFieldType::BinaryAttachment: + { + Builder << '"'; + Accessor.AsAttachment().ToHexString(Builder); + Builder << '"'; + } + break; + case CbFieldType::Hash: + { + Builder << '"'; + Accessor.AsHash().ToHexString(Builder); + Builder << '"'; + } + break; + case CbFieldType::Uuid: + { + Builder << '"'; + Accessor.AsUuid().ToString(Builder); + Builder << '"'; + } + break; + case CbFieldType::DateTime: + Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; + break; + case CbFieldType::TimeSpan: + { + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << '"' << Span.ToString("%h:%m:%s.%n") << '"'; + } + else + { + Builder << '"' << Span.ToString("%d.%h:%m:%s.%n") << '"'; + } + break; + } + case CbFieldType::ObjectId: + Builder << '"'; + Accessor.AsObjectId().ToString(Builder); + Builder << '"'; + break; + case CbFieldType::CustomById: + { + CbCustomById Custom = Accessor.AsCustomById(); + Builder << "{ \"Id\": "; + Builder << Custom.Id; + Builder << ", \"Data\": "; + AppendBase64String(Custom.Data); + Builder << " }"; + break; + } + case CbFieldType::CustomByName: + { + CbCustomByName Custom = Accessor.AsCustomByName(); + Builder << "{ \"Name\": "; + AppendQuotedString(Custom.Name); + Builder << ", \"Data\": "; + AppendBase64String(Custom.Data); + Builder << " }"; + break; + } + default: + ZEN_ASSERT(false); + break; + } + + NeedsComma = true; + NeedsNewLine = true; + } + +private: + void WriteOptionalComma() + { + if (NeedsComma) + { + NeedsComma = false; + Builder << ','; + } + } + + void WriteOptionalNewLine() + { + if (NeedsNewLine) + { + NeedsNewLine = false; + Builder << NewLineAndIndent; + } + } + + void AppendQuotedString(std::u8string_view Value) + { + using namespace std::literals; + + const AsciiSet EscapeSet( + "\\\"\b\f\n\r\t" + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"); + + Builder << '\"'; + while (!Value.empty()) + { + std::u8string_view Verbatim = AsciiSet::FindPrefixWithout(Value, EscapeSet); + Builder << Verbatim; + + Value = Value.substr(Verbatim.size()); + + std::u8string_view Escape = AsciiSet::FindPrefixWith(Value, EscapeSet); + for (char Char : Escape) + { + switch (Char) + { + case '\\': + Builder << "\\\\"sv; + break; + case '\"': + Builder << "\\\""sv; + break; + case '\b': + Builder << "\\b"sv; + break; + case '\f': + Builder << "\\f"sv; + break; + case '\n': + Builder << "\\n"sv; + break; + case '\r': + Builder << "\\r"sv; + break; + case '\t': + Builder << "\\t"sv; + break; + default: + Builder << Char; + break; + } + } + Value = Value.substr(Escape.size()); + } + Builder << '\"'; + } + + void AppendBase64String(MemoryView Value) + { + Builder << '"'; + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + } + +private: + StringBuilderBase& Builder; + StringBuilder<32> NewLineAndIndent; + bool NeedsComma{false}; + bool NeedsNewLine{false}; +}; + +void +CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) +{ + CbJsonWriter Writer(Builder); + Writer.WriteField(Object.AsFieldView()); +} + +////////////////////////////////////////////////////////////////////////// + #if ZEN_WITH_TESTS void uson_forcelink() @@ -1294,6 +1806,32 @@ TEST_CASE("uson.null") CHECK(Field.IsNull() == false); } } + +TEST_CASE("uson.json") +{ + SUBCASE("string") + { + CbObjectWriter Writer; + Writer << "KeyOne" + << "ValueOne"; + Writer << "KeyTwo" + << "ValueTwo"; + CbObject Obj = Writer.Save(); + + StringBuilder<128> Sb; + const std::string_view JsonText = Obj.ToJson(Sb).ToView(); + + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText.data(), JsonError); + + const std::string ValueOne = Json["KeyOne"].string_value(); + const std::string ValueTwo = Json["KeyTwo"].string_value(); + + CHECK(JsonError.empty()); + CHECK(ValueOne == "ValueOne"); + CHECK(ValueTwo == "ValueTwo"); + } +} #endif } // namespace zen diff --git a/zencore/include/zencore/base64.h b/zencore/include/zencore/base64.h new file mode 100644 index 000000000..4d78b085f --- /dev/null +++ b/zencore/include/zencore/base64.h @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencore.h" + +namespace zen { + +struct Base64 +{ + template + static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest); + + static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; } +}; + +} // namespace zen diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index 60e9ec7aa..ab01402f8 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -42,8 +42,22 @@ public: } inline uint64_t GetTicks() const { return Ticks; } - inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; } - inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; } + + int GetYear() const; + int GetMonth() const; + int GetDay() const; + int GetHour() const; + int GetHour12() const; + int GetMinute() const; + int GetSecond() const; + int GetMillisecond() const; + void GetDate(int& Year, int& Month, int& Day) const; + + inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; } + inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; } + + std::string ToString(const char* Format) const; + std::string ToIso8601() const; private: void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond); @@ -99,6 +113,25 @@ public: /** The number of timespan ticks per year (365 days, not accounting for leap years). */ static constexpr int64_t TicksPerYear = 365 * TicksPerDay; + int GetFractionTicks() const { return (int)(Ticks % TicksPerSecond); } + + int GetFractionMicro() const { return (int)((Ticks % TicksPerSecond) / TicksPerMicrosecond); } + + int GetFractionMilli() const { return (int)((Ticks % TicksPerSecond) / TicksPerMillisecond); } + + int GetFractionNano() const { return (int)((Ticks % TicksPerSecond) * NanosecondsPerTick); } + + int GetDays() const { return (int)(Ticks / TicksPerDay); } + + int GetHours() const { return (int)((Ticks / TicksPerHour) % 24); } + + int GetMinutes() const { return (int)((Ticks / TicksPerMinute) % 60); } + + int GetSeconds() const { return (int)((Ticks / TicksPerSecond) % 60); } + + ZENCORE_API std::string ToString(const char* Format) const; + ZENCORE_API std::string ToString() const; + private: void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano); @@ -108,6 +141,8 @@ private: struct Guid { uint32_t A, B, C, D; + + StringBuilderBase& ToString(StringBuilderBase& OutString) const; }; ////////////////////////////////////////////////////////////////////////// @@ -442,6 +477,11 @@ public: /** Returns the name of the field if it has a name, otherwise an empty view. */ constexpr inline std::string_view GetName() const { return std::string_view(static_cast(Payload) - NameLen, NameLen); } + /** Returns the name of the field if it has a name, otherwise an empty view. */ + constexpr inline std::u8string_view GetU8Name() const + { + return std::u8string_view(static_cast(Payload) - NameLen, NameLen); + } /** Returns the value for unchecked access. Prefer the typed accessors below. */ inline CbValue GetValue() const; @@ -907,6 +947,11 @@ private: inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {} }; +/** + * Serialize a compact binary object to JSON. + */ +ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder); + class CbObjectView : protected CbFieldView { friend class CbFieldView; @@ -989,6 +1034,12 @@ public: /** Whether the field has a value. */ using CbFieldView::operator bool; + StringBuilderBase& ToJson(StringBuilderBase& Builder) const + { + CompactBinaryToJson(*this, Builder); + return Builder; + } + private: friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); } friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); } diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h index bb9b1c896..2c0d10577 100644 --- a/zencore/include/zencore/string.h +++ b/zencore/include/zencore/string.h @@ -14,6 +14,8 @@ #include #include +#include + namespace zen { ////////////////////////////////////////////////////////////////////////// @@ -94,6 +96,14 @@ public: const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete; const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete; + inline size_t AddUninitialized(size_t Count) + { + EnsureCapacity(Count); + const size_t OldCount = Size(); + m_CurPos += Count; + return OldCount; + } + StringBuilderImpl& Append(C OneChar) { EnsureCapacity(1); @@ -209,6 +219,12 @@ public: return AppendRange(String.data(), String.data() + String.size()); } + inline void RemoveSuffix(int32_t Count) + { + ZEN_ASSERT(Count <= Size()); + m_CurPos -= Count; + } + inline const C* c_str() const { EnsureNulTerminated(); @@ -322,6 +338,12 @@ protected: extern template class StringBuilderImpl; +inline StringBuilderImpl& +operator<<(StringBuilderImpl& Builder, char Char) +{ + return Builder.Append(Char); +} + class StringBuilderBase : public StringBuilderImpl { public: @@ -661,6 +683,303 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func) ////////////////////////////////////////////////////////////////////////// +/** + * ASCII character bitset useful for fast and readable parsing + * + * Entirely constexpr. Works with both wide and narrow strings. + * + * Example use cases: + * + * constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n"); + * bool bIsWhitespace = WhitespaceCharacters.Test(MyChar); + * const char* HelloWorld = AsciiSet::Skip(" \t\tHello world!", WhitespaceCharacters); + * + * constexpr AsciiSet XmlEscapeChars("&<>\"'"); + * check(AsciiSet::HasNone(EscapedXmlString, XmlEscapeChars)); + * + * constexpr AsciiSet Delimiters(".:;"); + * const TCHAR* DelimiterOrEnd = AsciiSet::FindFirstOrEnd(PrefixedName, Delimiters); + * FString Prefix(PrefixedName, DelimiterOrEnd - PrefixedName); + * + * constexpr AsciiSet Slashes("/\\"); + * const TCHAR* SlashOrEnd = AsciiSet::FindLastOrEnd(PathName, Slashes); + * const TCHAR* FileName = *SlashOrEnd ? SlashOrEnd + 1 : PathName; + */ +class AsciiSet +{ +public: + template + constexpr AsciiSet(const CharType (&Chars)[N]) : AsciiSet(StringToBitset(Chars)) + { + } + + /** Returns true if a character is part of the set */ + template + constexpr inline bool Contains(CharType Char) const + { + using UnsignedCharType = std::make_unsigned::type; + + return !!TestImpl((UnsignedCharType)Char); + } + + /** Returns non-zero if a character is part of the set. Prefer Contains() to avoid VS2019 conversion warnings. */ + template + constexpr inline uint64_t Test(CharType Char) const + { + using UnsignedCharType = std::make_unsigned::type; + + return TestImpl((UnsignedCharType)Char); + } + + /** Create new set with specified character in it */ + constexpr inline AsciiSet operator+(char Char) const + { + using UnsignedCharType = std::make_unsigned::type; + + InitData Bitset = {LoMask, HiMask}; + SetImpl(Bitset, (UnsignedCharType)Char); + return AsciiSet(Bitset); + } + + /** Create new set containing inverse set of characters - likely including null-terminator */ + constexpr inline AsciiSet operator~() const { return AsciiSet(~LoMask, ~HiMask); } + + ////////// Algorithms for C strings ////////// + + /** Find first character of string inside set or end pointer. Never returns null. */ + template + static constexpr const CharType* FindFirstOrEnd(const CharType* Str, AsciiSet Set) + { + for (AsciiSet SetOrNil(Set.LoMask | NilMask, Set.HiMask); !SetOrNil.Test(*Str); ++Str) + ; + + return Str; + } + + /** Find last character of string inside set or end pointer. Never returns null. */ + template + static constexpr const CharType* FindLastOrEnd(const CharType* Str, AsciiSet Set) + { + const CharType* Last = FindFirstOrEnd(Str, Set); + + for (const CharType* It = Last; *It; It = FindFirstOrEnd(It + 1, Set)) + { + Last = It; + } + + return Last; + } + + /** Find first character of string outside of set. Never returns null. */ + template + static constexpr const CharType* Skip(const CharType* Str, AsciiSet Set) + { + while (Set.Contains(*Str)) + { + ++Str; + } + + return Str; + } + + /** Test if string contains any character in set */ + template + static constexpr bool HasAny(const CharType* Str, AsciiSet Set) + { + return *FindFirstOrEnd(Str, Set) != '\0'; + } + + /** Test if string contains no character in set */ + template + static constexpr bool HasNone(const CharType* Str, AsciiSet Set) + { + return *FindFirstOrEnd(Str, Set) == '\0'; + } + + /** Test if string contains any character outside of set */ + template + static constexpr bool HasOnly(const CharType* Str, AsciiSet Set) + { + return *Skip(Str, Set) == '\0'; + } + + ////////// Algorithms for string types like FStringView and FString ////////// + + /** Get initial substring with all characters in set */ + template + static constexpr StringType FindPrefixWith(const StringType& Str, AsciiSet Set) + { + return Scan(Str, Set); + } + + /** Get initial substring with no characters in set */ + template + static constexpr StringType FindPrefixWithout(const StringType& Str, AsciiSet Set) + { + return Scan(Str, Set); + } + + /** Trim initial characters in set */ + template + static constexpr StringType TrimPrefixWith(const StringType& Str, AsciiSet Set) + { + return Scan(Str, Set); + } + + /** Trim initial characters not in set */ + template + static constexpr StringType TrimPrefixWithout(const StringType& Str, AsciiSet Set) + { + return Scan(Str, Set); + } + + /** Get trailing substring with all characters in set */ + template + static constexpr StringType FindSuffixWith(const StringType& Str, AsciiSet Set) + { + return Scan(Str, Set); + } + + /** Get trailing substring with no characters in set */ + template + static constexpr StringType FindSuffixWithout(const StringType& Str, AsciiSet Set) + { + return Scan(Str, Set); + } + + /** Trim trailing characters in set */ + template + static constexpr StringType TrimSuffixWith(const StringType& Str, AsciiSet Set) + { + return Scan(Str, Set); + } + + /** Trim trailing characters not in set */ + template + static constexpr StringType TrimSuffixWithout(const StringType& Str, AsciiSet Set) + { + return Scan(Str, Set); + } + + /** Test if string contains any character in set */ + template + static constexpr bool HasAny(const StringType& Str, AsciiSet Set) + { + return !HasNone(Str, Set); + } + + /** Test if string contains no character in set */ + template + static constexpr bool HasNone(const StringType& Str, AsciiSet Set) + { + uint64_t Match = 0; + for (auto Char : Str) + { + Match |= Set.Test(Char); + } + return Match == 0; + } + + /** Test if string contains any character outside of set */ + template + static constexpr bool HasOnly(const StringType& Str, AsciiSet Set) + { + auto End = Str.data() + Str.size(); + return FindFirst(Set, GetData(Str), End) == End; + } + +private: + enum class EDir + { + Forward, + Reverse + }; + enum class EInclude + { + Members, + NonMembers + }; + enum class EKeep + { + Head, + Tail + }; + + template + static constexpr const CharType* FindFirst(AsciiSet Set, const CharType* It, const CharType* End) + { + for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); ++It) + ; + return It; + } + + template + static constexpr const CharType* FindLast(AsciiSet Set, const CharType* It, const CharType* End) + { + for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); --It) + ; + return It; + } + + template + static constexpr StringType Scan(const StringType& Str, AsciiSet Set) + { + auto Begin = Str.data(); + auto End = Begin + Str.size(); + auto It = Dir == EDir::Forward ? FindFirst(Set, Begin, End) : FindLast(Set, End - 1, Begin - 1) + 1; + + return Keep == EKeep::Head ? StringType(Begin, static_cast(It - Begin)) : StringType(It, static_cast(End - It)); + } + + // Work-around for constexpr limitations + struct InitData + { + uint64_t Lo, Hi; + }; + static constexpr uint64_t NilMask = uint64_t(1) << '\0'; + + static constexpr inline void SetImpl(InitData& Bitset, uint32_t Char) + { + uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0); + uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1); + uint64_t Bit = uint64_t(1) << uint8_t(Char & 0x3f); + + Bitset.Lo |= Bit & IsLo; + Bitset.Hi |= Bit & IsHi; + } + + constexpr inline uint64_t TestImpl(uint32_t Char) const + { + uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0); + uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1); + uint64_t Bit = uint64_t(1) << (Char & 0x3f); + + return (Bit & IsLo & LoMask) | (Bit & IsHi & HiMask); + } + + template + static constexpr InitData StringToBitset(const CharType (&Chars)[N]) + { + using UnsignedCharType = std::make_unsigned::type; + + InitData Bitset = {0, 0}; + for (int I = 0; I < N - 1; ++I) + { + SetImpl(Bitset, UnsignedCharType(Chars[I])); + } + + return Bitset; + } + + constexpr AsciiSet(InitData Bitset) : LoMask(Bitset.Lo), HiMask(Bitset.Hi) {} + + constexpr AsciiSet(uint64_t Lo, uint64_t Hi) : LoMask(Lo), HiMask(Hi) {} + + uint64_t LoMask, HiMask; +}; + +////////////////////////////////////////////////////////////////////////// + void string_forcelink(); // internal } // namespace zen diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index f6093cb96..4b9c1af1b 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -102,9 +102,11 @@ // Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable. # define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name)) # define ZEN_FORCENOINLINE __declspec(noinline) /* Force code to NOT be inline */ +# define LINE_TERMINATOR_ANSI "\r\n" #else # define ZEN_CODE_SECTION(Name) # define ZEN_FORCENOINLINE +# define LINE_TERMINATOR_ANSI "\n" #endif #if ZEN_ARCH_ARM64 diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj index 2322f7173..3adf779ed 100644 --- a/zencore/zencore.vcxproj +++ b/zencore/zencore.vcxproj @@ -113,6 +113,7 @@ + @@ -157,6 +158,7 @@ + diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters index d2e7a3159..92aa0db1d 100644 --- a/zencore/zencore.vcxproj.filters +++ b/zencore/zencore.vcxproj.filters @@ -44,6 +44,7 @@ + @@ -77,6 +78,7 @@ + diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp index d999f7f60..795e81ea8 100644 --- a/zenhttp/httpserver.cpp +++ b/zenhttp/httpserver.cpp @@ -319,9 +319,17 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbPackage Data) void HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbObject Data) { - SharedBuffer Buf = Data.GetBuffer(); - std::array Buffers{IoBufferBuilder::MakeCloneFromMemory(Buf.GetData(), Buf.GetSize())}; - return WriteResponse(ResponseCode, HttpContentType::kCbObject, Buffers); + if (m_AcceptType == HttpContentType::kJSON) + { + ExtendableStringBuilder<1024> Sb; + WriteResponse(ResponseCode, HttpContentType::kJSON, Data.ToJson(Sb).ToView()); + } + else + { + SharedBuffer Buf = Data.GetBuffer(); + std::array Buffers{IoBufferBuilder::MakeCloneFromMemory(Buf.GetData(), Buf.GetSize())}; + return WriteResponse(ResponseCode, HttpContentType::kCbObject, Buffers); + } } void -- cgit v1.2.3 From f6ee71a52da990b3bdc41bf56fcef87b30ca964b Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 11:31:43 +0200 Subject: Added simple compact binary endpoint for JSON testing --- zenserver/testing/httptest.cpp | 11 +++++++++++ zenserver/testing/httptest.h | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/zenserver/testing/httptest.cpp b/zenserver/testing/httptest.cpp index c4fd6003c..18d63a6ef 100644 --- a/zenserver/testing/httptest.cpp +++ b/zenserver/testing/httptest.cpp @@ -2,6 +2,7 @@ #include "httptest.h" +#include #include namespace zen { @@ -13,6 +14,16 @@ HttpTestingService::HttpTestingService() [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); }, HttpVerb::kGet); + m_Router.RegisterRoute( + "json", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Obj.AddInteger("counter", ++m_Counter); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + m_Router.RegisterRoute( "echo", [this](HttpRouterRequest& Req) { diff --git a/zenserver/testing/httptest.h b/zenserver/testing/httptest.h index b445fb450..f55780d05 100644 --- a/zenserver/testing/httptest.h +++ b/zenserver/testing/httptest.h @@ -5,6 +5,8 @@ #include #include +#include + namespace zen { /** @@ -37,7 +39,8 @@ public: }; private: - HttpRequestRouter m_Router; + HttpRequestRouter m_Router; + std::atomic m_Counter{0}; RwLock m_RwLock; std::unordered_map> m_HandlerMap; -- cgit v1.2.3 From 231a720c099ad088c53316d03cb2b6348d2cf2a7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 11:32:00 +0200 Subject: clang-format --- zenstore/cidstore.cpp | 2 +- zenstore/include/zenstore/cidstore.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index d76058bd1..df5c32d25 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -46,7 +46,7 @@ struct CidStore::Impl RwLock::ExclusiveLockScope _(m_Lock); - auto It = m_CidMap.try_emplace(DecompressedId, Compressed); + auto It = m_CidMap.try_emplace(DecompressedId, Compressed); if (!It.second) { if (It.first.value() != Compressed) diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 2eea04164..5f567e7fc 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -25,9 +25,9 @@ class IoBuffer; * be used to deal with other kinds of indirections in the future. For example, if we want * to support chunking then a CID may represent a list of chunks which could be concatenated * to form the referenced chunk. - * + * * It would likely be possible to implement this mapping in a more efficient way if we - * integrate it into the CAS store itself, so we can avoid maintaining copies of large + * integrate it into the CAS store itself, so we can avoid maintaining copies of large * hashes in multiple locations. This would also allow us to consolidate commit logs etc * which would be more resilient than the current split log scheme * -- cgit v1.2.3 From dd9aec560688547dfd60090436d843c76450bd62 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 11:43:40 +0200 Subject: GetWindowsErrorAsString() -> GetSystemErrorAsString() --- zencore/except.cpp | 4 ++-- zencore/include/zencore/except.h | 2 +- zenhttp/httpsys.cpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zencore/except.cpp b/zencore/except.cpp index 44b8edffb..8614b52e1 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -72,11 +72,11 @@ ThrowSystemError(uint32_t ErrorCode, std::string_view Message) std::string GetLastErrorAsString() { - return GetWindowsErrorAsString(zen::GetLastError()); + return GetSystemErrorAsString(zen::GetLastError()); } std::string -GetWindowsErrorAsString(uint32_t Win32ErrorCode) +GetSystemErrorAsString(uint32_t Win32ErrorCode) { return std::error_code(Win32ErrorCode, std::system_category()).message(); } diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 5cfefb1e2..3430565d2 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -27,7 +27,7 @@ ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const st ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); ZENCORE_API std::string GetLastErrorAsString(); -ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); +ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode); inline int32_t GetLastError() diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 2a50388e3..b08e39a8a 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -417,7 +417,7 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB if (IoResult != NO_ERROR) { - ZEN_WARN("response aborted due to error: '{}'", GetWindowsErrorAsString(IoResult)); + ZEN_WARN("response aborted due to error: '{}'", GetSystemErrorAsString(IoResult)); // if one transmit failed there's really no need to go on return nullptr; @@ -554,7 +554,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) CancelThreadpoolIo(Iocp); ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}', request id: {}", - GetWindowsErrorAsString(SendResult), + GetSystemErrorAsString(SendResult), HttpReq->pRawUrl, HttpReq->RequestId); -- cgit v1.2.3