From c07e8c760ed56db387b6612ed8392d6e78301479 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Tue, 7 Sep 2021 15:11:11 +0200 Subject: Return stats from Zen/Jupiter HTTP client. --- zenserver/upstream/jupiter.cpp | 33 +++---- zenserver/upstream/jupiter.h | 6 +- zenserver/upstream/upstreamcache.cpp | 183 +++++++++++++++++++++++++++++++---- zenserver/upstream/upstreamcache.h | 6 +- zenserver/upstream/zen.cpp | 12 +-- zenserver/upstream/zen.h | 4 +- 6 files changed, 194 insertions(+), 50 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 9a013963a..815e9014e 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -80,15 +80,12 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke Session.SetOption(cpr::Header{{"Authorization", Auth}}); cpr::Response Response = Session.Get(); - m_Log.debug("GET {}", Response); + + const bool Success = Response.status_code == 200; + const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - if (Response.status_code == 200) - { - return {IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()), true}; - } - - return {.Success = false}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -116,12 +113,10 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte cpr::Response Response = Session.Get(); m_Log.debug("GET {}", Response); - if (Response.status_code == 200) - { - return {IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()), true}; - } + const bool Success = Response.status_code == 200; + const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Success = false}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -141,12 +136,10 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) cpr::Response Response = Session.Get(); m_Log.debug("GET {}", Response); - if (Response.status_code == 200) - { - return {IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()), true}; - } + const bool Success = Response.status_code == 200; + const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Success = false}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -170,7 +163,7 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Ke cpr::Response Response = Session.Put(); m_Log.debug("PUT {}", Response); - return {.Success = Response.status_code == 200}; + return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } CloudCacheResult @@ -201,7 +194,7 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer cpr::Response Response = Session.Put(); m_Log.debug("PUT {}", Response); - return {.Success = Response.status_code == 200}; + return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } CloudCacheResult @@ -222,7 +215,7 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) cpr::Response Response = Session.Put(); m_Log.debug("PUT {}", Response); - return {.Success = Response.status_code == 200}; + return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } std::vector diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index f3af88e77..ca60804dc 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -43,8 +43,10 @@ private: struct CloudCacheResult { - IoBuffer Value; - bool Success = false; + IoBuffer Response; + int64_t Bytes = 0; + double ElapsedSeconds = 0.0; + bool Success = false; }; /** diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index f7a91acb5..04ac40a39 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -16,8 +16,10 @@ #include #include +#include #include #include +#include namespace zen { @@ -122,7 +124,10 @@ namespace detail { Result = Session.GetRef(CacheKey.Bucket, CacheKey.Hash, Type); } - return {.Value = Result.Value, .Success = Result.Success}; + return {.Value = Result.Response, + .Bytes = Result.Bytes, + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}; } catch (std::exception& e) { @@ -136,7 +141,11 @@ namespace detail { { zen::CloudCacheSession Session(m_Client); const CloudCacheResult Result = Session.GetCompressedBlob(PayloadKey.PayloadId); - return {.Value = Result.Value, .Success = Result.Success}; + + return {.Value = Result.Response, + .Bytes = Result.Bytes, + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}; } catch (std::exception& e) { @@ -173,10 +182,13 @@ namespace detail { } } - return {.Success = Result.Success}; + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}; } else { + int64_t TotalBytes = 0; + double TotalElapsedSeconds = 0.0; + for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) { CloudCacheResult Result; @@ -185,24 +197,29 @@ namespace detail { Result = Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); } + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) { - return {.Reason = "Failed to upload payload", .Success = false}; + return {.Reason = "Failed to upload payload", + .Bytes = TotalBytes, + .ElapsedSeconds = TotalElapsedSeconds, + .Success = false}; } } + CloudCacheResult Result; + for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { - CloudCacheResult Result; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) - { - Result = Session.PutRef(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - RecordValue, - ZenContentType::kCbObject); - } - - return {.Success = Result.Success}; + Result = + Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, ZenContentType::kCbObject); } + + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + + return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = Result.Success}; } } catch (std::exception& e) @@ -243,7 +260,11 @@ namespace detail { { ZenStructuredCacheSession Session(*m_Client); const ZenCacheResult Result = Session.GetCacheRecord(CacheKey.Bucket, CacheKey.Hash, Type); - return {.Value = Result.Response, .Success = Result.Success}; + + return {.Value = Result.Response, + .Bytes = Result.Bytes, + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}; } catch (std::exception& e) { @@ -258,7 +279,11 @@ namespace detail { ZenStructuredCacheSession Session(*m_Client); const ZenCacheResult Result = Session.GetCachePayload(PayloadKey.CacheKey.Bucket, PayloadKey.CacheKey.Hash, PayloadKey.PayloadId); - return {.Value = Result.Response, .Success = Result.Success}; + + return {.Value = Result.Response, + .Bytes = Result.Bytes, + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}; } catch (std::exception& e) { @@ -277,6 +302,8 @@ namespace detail { { zen::ZenStructuredCacheSession Session(*m_Client); ZenCacheResult Result; + int64_t TotalBytes = 0; + double TotalElapsedSeconds = 0.0; for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) { @@ -289,9 +316,15 @@ namespace detail { Payloads[Idx]); } + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) { - return {.Reason = "Failed to upload payload", .Success = false}; + return {.Reason = "Failed to upload payload", + .Bytes = TotalBytes, + .ElapsedSeconds = TotalElapsedSeconds, + .Success = false}; } } @@ -301,7 +334,10 @@ namespace detail { Result = Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, CacheRecord.Type); } - return {.Success = Result.Success}; + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + + return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = Result.Success}; } catch (std::exception& e) { @@ -318,6 +354,108 @@ namespace detail { ////////////////////////////////////////////////////////////////////////// +class UpstreamStats final +{ + using clock = std::chrono::steady_clock; + using time_point = std::chrono::time_point; + + struct StatCounters + { + int64_t Bytes = {}; + int64_t Count = {}; + double Seconds = {}; + }; + + struct EndpointStats + { + mutable std::mutex Lock; + std::unordered_map Counters; + }; + +public: + UpstreamStats() : m_Log(zen::logging::Get("upstreamstats")) { m_LastTime = clock::now(); } + + void Add(const UpstreamEndpoint& Endpoint, const GetUpstreamCacheResult& Result) + { + std::unique_lock Lock(m_DownStats.Lock); + + auto& Counters = m_DownStats.Counters[&Endpoint]; + Counters.Bytes += Result.Bytes; + Counters.Seconds += Result.ElapsedSeconds; + Counters.Count++; + + LogStats(); + } + + void Add(const UpstreamEndpoint& Endpoint, const PutUpstreamCacheResult& Result) + { + std::unique_lock Lock(m_UpStats.Lock); + + auto& Counters = m_UpStats.Counters[&Endpoint]; + Counters.Bytes += Result.Bytes; + Counters.Seconds += Result.ElapsedSeconds; + Counters.Count++; + + LogStats(); + } + +private: + void LogStats() + { + using std::chrono::duration_cast; + using std::chrono::seconds; + + const seconds Duration = duration_cast(clock::now() - m_LastTime); + if (Duration < seconds(5)) + { + return; + } + + { + std::unique_lock Lock(m_DownStats.Lock); + for (const auto& Kv : m_DownStats.Counters) + { + const UpstreamEndpoint& Endpoint = *Kv.first; + const StatCounters& Counters = Kv.second; + + m_Log.info("Endpoint: {}, Total downloaded: {}Kb, Total time: {}s, Count: {}, Speed: {} Kb/s, Avg: {} (Req/s)", + Kv.first->DisplayName(), + double(Counters.Bytes) / 1024.0, + Counters.Seconds, + Counters.Count, + (double(Counters.Bytes) / 1024.0) / Counters.Seconds, + double(Counters.Count) / Counters.Seconds); + } + } + + { + std::unique_lock Lock(m_UpStats.Lock); + for (const auto& Kv : m_UpStats.Counters) + { + const UpstreamEndpoint& Endpoint = *Kv.first; + const StatCounters& Counters = Kv.second; + + m_Log.info("Endpoint: {}, Total uploaded: {}Kb, Total time: {}s, Count: {}, Speed: {} Kb/s, Avg: {} (Req/s)", + Kv.first->DisplayName(), + double(Counters.Bytes) / 1024.0, + Counters.Seconds, + Counters.Count, + (double(Counters.Bytes) / 1024.0) / Counters.Seconds, + double(Counters.Count) / Counters.Seconds); + } + } + + m_LastTime = clock::now(); + } + + spdlog::logger& m_Log; + EndpointStats m_UpStats; + EndpointStats m_DownStats; + time_point m_LastTime = {}; +}; + +////////////////////////////////////////////////////////////////////////// + class DefaultUpstreamCache final : public UpstreamCache { public: @@ -362,6 +500,7 @@ public: { if (GetUpstreamCacheResult Result = Endpoint->GetCacheRecord(CacheKey, Type); Result.Success) { + m_Stats.Add(*Endpoint, Result); return Result; } } @@ -375,6 +514,7 @@ public: { if (GetUpstreamCacheResult Result = Endpoint->GetCachePayload(PayloadKey); Result.Success) { + m_Stats.Add(*Endpoint, Result); return Result; } } @@ -425,7 +565,11 @@ private: for (auto& Endpoint : m_Endpoints) { - Endpoint->PutCacheRecord(CacheRecord, CacheValue.Value, std::span(Payloads)); + if (PutUpstreamCacheResult Result = Endpoint->PutCacheRecord(CacheRecord, CacheValue.Value, std::span(Payloads)); + Result.Success) + { + m_Stats.Add(*Endpoint, Result); + } } } @@ -477,6 +621,7 @@ private: ::ZenCacheStore& m_CacheStore; CidStore& m_CidStore; UpstreamQueue m_UpstreamQueue; + UpstreamStats m_Stats; std::vector> m_Endpoints; std::vector m_UpstreamThreads; std::atomic_bool m_IsRunning{false}; diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index d8359bc2c..d2344c9bd 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -43,12 +43,16 @@ struct GetUpstreamCacheResult { IoBuffer Value; std::string Reason; - bool Success = false; + int64_t Bytes = 0; + double ElapsedSeconds = 0.0; + bool Success = false; }; struct PutUpstreamCacheResult { std::string Reason; + int64_t Bytes = 0; + double ElapsedSeconds = 0.0; bool Success; }; diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 7ed4eead0..8df2f2104 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -416,12 +416,10 @@ ZenStructuredCacheSession::GetCachePayload(std::string_view BucketId, const IoHa cpr::Response Response = Session.Get(); m_Log.debug("GET {}", Response); - if (Response.status_code == 200) - { - return {.Response = IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()), .Success = true}; - } + const bool Success = Response.status_code == 200; + const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } ZenCacheResult @@ -440,7 +438,7 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view BucketId, const IoHas cpr::Response Response = Session.Put(); m_Log.debug("PUT {}", Response); - return {.Success = Response.status_code == 200}; + return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } ZenCacheResult @@ -458,7 +456,7 @@ ZenStructuredCacheSession::PutCachePayload(std::string_view BucketId, const IoHa cpr::Response Response = Session.Put(); m_Log.debug("PUT {}", Response); - return {.Success = Response.status_code == 200}; + return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } } // namespace zen diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index 5df6da4a3..849c67bc4 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -88,7 +88,9 @@ namespace detail { struct ZenCacheResult { IoBuffer Response; - bool Success = false; + int64_t Bytes = 0; + double ElapsedSeconds = 0.0; + bool Success = false; }; /** Zen Structured Cache session -- cgit v1.2.3 From 6356b19b0f717f40d388b930c63d4811e757fe50 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Tue, 7 Sep 2021 20:01:06 +0200 Subject: Updated upstream stats calculation. --- zenserver/upstream/upstreamcache.cpp | 83 ++++++++++++++---------------------- zenserver/upstream/zen.cpp | 8 ++-- 2 files changed, 34 insertions(+), 57 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 04ac40a39..c1002a6da 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -16,8 +16,8 @@ #include #include -#include #include +#include #include #include @@ -356,8 +356,7 @@ namespace detail { class UpstreamStats final { - using clock = std::chrono::steady_clock; - using time_point = std::chrono::time_point; + static constexpr uint64_t MaxSampleCount = 100ull; struct StatCounters { @@ -366,14 +365,16 @@ class UpstreamStats final double Seconds = {}; }; + using StatsMap = std::unordered_map; + struct EndpointStats { - mutable std::mutex Lock; - std::unordered_map Counters; + mutable std::mutex Lock; + StatsMap Counters; }; public: - UpstreamStats() : m_Log(zen::logging::Get("upstreamstats")) { m_LastTime = clock::now(); } + UpstreamStats() : m_Log(zen::logging::Get("upstream")) {} void Add(const UpstreamEndpoint& Endpoint, const GetUpstreamCacheResult& Result) { @@ -384,7 +385,11 @@ public: Counters.Seconds += Result.ElapsedSeconds; Counters.Count++; - LogStats(); + if (Counters.Count >= MaxSampleCount) + { + LogStats("STATS - (downstream):"sv, m_DownStats.Counters); + Counters = StatCounters{}; + } } void Add(const UpstreamEndpoint& Endpoint, const PutUpstreamCacheResult& Result) @@ -396,62 +401,36 @@ public: Counters.Seconds += Result.ElapsedSeconds; Counters.Count++; - LogStats(); + if (Counters.Count >= MaxSampleCount) + { + LogStats("STATS - (upstream):"sv, m_UpStats.Counters); + Counters = StatCounters{}; + } } private: - void LogStats() + void LogStats(std::string_view What, const std::unordered_map& EndpointStats) { - using std::chrono::duration_cast; - using std::chrono::seconds; - - const seconds Duration = duration_cast(clock::now() - m_LastTime); - if (Duration < seconds(5)) + for (const auto& Kv : EndpointStats) { - return; + const UpstreamEndpoint& Endpoint = *Kv.first; + const StatCounters& Counters = Kv.second; + const double TotalMb = double(Counters.Bytes) / 1024.0 / 1024.0; + + m_Log.info("{} Endpoint: {}, Bytes: {:.2f} MB, Time: {:.2f} s, Speed: {:.2f} MB/s, Avg: {:.2f} ms/request, Samples: {}", + What, + Kv.first->DisplayName(), + TotalMb, + Counters.Seconds, + TotalMb / Counters.Seconds, + (Counters.Seconds * 1000.0) / double(Counters.Count), + Counters.Count); } - - { - std::unique_lock Lock(m_DownStats.Lock); - for (const auto& Kv : m_DownStats.Counters) - { - const UpstreamEndpoint& Endpoint = *Kv.first; - const StatCounters& Counters = Kv.second; - - m_Log.info("Endpoint: {}, Total downloaded: {}Kb, Total time: {}s, Count: {}, Speed: {} Kb/s, Avg: {} (Req/s)", - Kv.first->DisplayName(), - double(Counters.Bytes) / 1024.0, - Counters.Seconds, - Counters.Count, - (double(Counters.Bytes) / 1024.0) / Counters.Seconds, - double(Counters.Count) / Counters.Seconds); - } - } - - { - std::unique_lock Lock(m_UpStats.Lock); - for (const auto& Kv : m_UpStats.Counters) - { - const UpstreamEndpoint& Endpoint = *Kv.first; - const StatCounters& Counters = Kv.second; - - m_Log.info("Endpoint: {}, Total uploaded: {}Kb, Total time: {}s, Count: {}, Speed: {} Kb/s, Avg: {} (Req/s)", - Kv.first->DisplayName(), - double(Counters.Bytes) / 1024.0, - Counters.Seconds, - Counters.Count, - (double(Counters.Bytes) / 1024.0) / Counters.Seconds, - double(Counters.Count) / Counters.Seconds); - } - } - - m_LastTime = clock::now(); } spdlog::logger& m_Log; EndpointStats m_UpStats; EndpointStats m_DownStats; - time_point m_LastTime = {}; }; ////////////////////////////////////////////////////////////////////////// diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 8df2f2104..7cdaa0036 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -394,12 +394,10 @@ ZenStructuredCacheSession::GetCacheRecord(std::string_view BucketId, const IoHas cpr::Response Response = Session.Get(); m_Log.debug("GET {}", Response); - if (Response.status_code == 200) - { - return {.Response = IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()), .Success = true}; - } + const bool Success = Response.status_code == 200; + const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } ZenCacheResult -- cgit v1.2.3 From e0f6f284fc3bb0996628db058130760ae1d650c6 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Tue, 7 Sep 2021 20:10:46 +0200 Subject: Minor fixup and formatting. --- zenserver/upstream/jupiter.h | 4 ++-- zenserver/upstream/upstreamcache.cpp | 4 ++-- zenserver/upstream/upstreamcache.h | 10 +++++----- zenserver/upstream/zen.h | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index ca60804dc..efe9d07ba 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -44,8 +44,8 @@ private: struct CloudCacheResult { IoBuffer Response; - int64_t Bytes = 0; - double ElapsedSeconds = 0.0; + int64_t Bytes = {}; + double ElapsedSeconds = {}; bool Success = false; }; diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index c1002a6da..9e566749a 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -186,7 +186,7 @@ namespace detail { } else { - int64_t TotalBytes = 0; + int64_t TotalBytes = 0ull; double TotalElapsedSeconds = 0.0; for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) @@ -302,7 +302,7 @@ namespace detail { { zen::ZenStructuredCacheSession Session(*m_Client); ZenCacheResult Result; - int64_t TotalBytes = 0; + int64_t TotalBytes = 0ull; double TotalElapsedSeconds = 0.0; for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index d2344c9bd..327778452 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -43,17 +43,17 @@ struct GetUpstreamCacheResult { IoBuffer Value; std::string Reason; - int64_t Bytes = 0; - double ElapsedSeconds = 0.0; + int64_t Bytes = {}; + double ElapsedSeconds = {}; bool Success = false; }; struct PutUpstreamCacheResult { std::string Reason; - int64_t Bytes = 0; - double ElapsedSeconds = 0.0; - bool Success; + int64_t Bytes = {}; + double ElapsedSeconds = {}; + bool Success = false; }; /** diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index 849c67bc4..c4bff8980 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -88,8 +88,8 @@ namespace detail { struct ZenCacheResult { IoBuffer Response; - int64_t Bytes = 0; - double ElapsedSeconds = 0.0; + int64_t Bytes = {}; + double ElapsedSeconds = {}; bool Success = false; }; -- cgit v1.2.3 From cd9579c48d691c14d8aa4e7119714ee11c58437a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 8 Sep 2021 16:14:40 +0200 Subject: Changed logging defaults to be more suitable for production * Default log level changes from debug to info * Default logging mode changes from sync to async --- zenserver/diag/logging.cpp | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) (limited to 'zenserver') diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 89ed4ac6a..a5a4edfc0 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -4,6 +4,8 @@ #include "config.h" +#include +#include #include #include #include @@ -191,9 +193,27 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) std::filesystem::path LogPath = GlobalOptions.DataDir / "logs/zenserver.log"; - spdlog::level::level_enum LogLevel = spdlog::level::debug; + bool IsAsync = true; + spdlog::level::level_enum LogLevel = spdlog::level::info; + + if (GlobalOptions.IsDebug) + { + LogLevel = spdlog::level::debug; + IsAsync = false; + } + + if (IsAsync) + { + const int QueueSize = 8192; + const int ThreadCount = 1; + spdlog::init_thread_pool(QueueSize, ThreadCount); + + auto AsyncLogger = spdlog::create_async("main"); + spdlog::set_default_logger(AsyncLogger); + } // Sinks + auto ConsoleSink = std::make_shared(); auto FileSink = std::make_shared(zen::WideToUtf8(LogPath.c_str()), /* truncate */ true); @@ -201,9 +221,11 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) auto DefaultLogger = spdlog::default_logger(); auto& Sinks = spdlog::default_logger()->sinks(); Sinks.clear(); + Sinks.push_back(ConsoleSink); Sinks.push_back(FileSink); DefaultLogger->set_level(LogLevel); + DefaultLogger->flush_on(spdlog::level::err); // Jupiter - only log HTTP traffic to file auto JupiterLogger = std::make_shared("jupiter", FileSink); @@ -225,12 +247,17 @@ ShutdownLogging() spdlog::shutdown(); } +std::once_flag ConsoleInitFlag; +std::shared_ptr ConLogger; + spdlog::logger& ConsoleLog() { - static auto ConLogger = spdlog::stdout_color_mt("console"); + std::call_once(ConsoleInitFlag, [&] { + ConLogger = spdlog::stdout_color_mt("console"); - ConLogger->set_pattern("%v"); + ConLogger->set_pattern("%v"); + }); return *ConLogger; } -- cgit v1.2.3 From e44740f9bf2846e69e8c01d7b1f7a7b8545af6b6 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 8 Sep 2021 19:10:59 +0200 Subject: Moved a bunch of logging code into zencore --- zenserver/config.cpp | 4 ++-- zenserver/diag/logging.cpp | 47 +++------------------------------------------- zenserver/diag/logging.h | 10 ---------- 3 files changed, 5 insertions(+), 56 deletions(-) (limited to 'zenserver') diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 904e613a4..665c567cc 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -178,14 +178,14 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z if (result.count("help")) { - ConsoleLog().info("{}", options.help()); + zen::logging::ConsoleLog().info("{}", options.help()); exit(0); } } catch (cxxopts::OptionParseException& e) { - ConsoleLog().error("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help()); + zen::logging::ConsoleLog().error("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help()); throw; } diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index a5a4edfc0..796a15d01 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -189,6 +189,8 @@ EnableVTMode() void InitializeLogging(const ZenServerOptions& GlobalOptions) { + zen::logging::InitializeLogging(); + EnableVTMode(); std::filesystem::path LogPath = GlobalOptions.DataDir / "logs/zenserver.log"; @@ -243,48 +245,5 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) void ShutdownLogging() { - spdlog::drop_all(); - spdlog::shutdown(); -} - -std::once_flag ConsoleInitFlag; -std::shared_ptr ConLogger; - -spdlog::logger& -ConsoleLog() -{ - std::call_once(ConsoleInitFlag, [&] { - ConLogger = spdlog::stdout_color_mt("console"); - - ConLogger->set_pattern("%v"); - }); - - return *ConLogger; + zen::logging::ShutdownLogging(); } - -namespace zen::logging { - -spdlog::logger& -Default() -{ - return *spdlog::default_logger(); -} - -spdlog::logger& -Get(std::string_view Name) -{ - std::shared_ptr Logger = spdlog::get(std::string(Name)); - if (!Logger) - { - Logger = std::make_shared(std::string(Name), - begin(spdlog::default_logger()->sinks()), - end(spdlog::default_logger()->sinks())); - - Logger->set_level(spdlog::default_logger()->level()); - spdlog::register_logger(Logger); - } - - return *Logger; -} - -} // namespace zen::logging diff --git a/zenserver/diag/logging.h b/zenserver/diag/logging.h index bc93898ad..8df49f842 100644 --- a/zenserver/diag/logging.h +++ b/zenserver/diag/logging.h @@ -8,13 +8,3 @@ struct ZenServerOptions; void InitializeLogging(const ZenServerOptions& GlobalOptions); void ShutdownLogging(); - -spdlog::logger& ConsoleLog(); - -namespace zen::logging { - -spdlog::logger& Default(); - -spdlog::logger& Get(std::string_view Name); - -} // namespace zen::logging -- cgit v1.2.3 From 891b85a08ff008febf267c581dd072b1cc1be395 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 8 Sep 2021 19:11:46 +0200 Subject: Mesh should not default to on --- zenserver/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 665c567cc..ddef83c02 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -85,7 +85,7 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z "m", "mesh", "Enable mesh network", - cxxopts::value(ServiceConfig.MeshEnabled)->default_value("true"), + cxxopts::value(ServiceConfig.MeshEnabled)->default_value("false"), ""); options.add_option("diagnostics", -- cgit v1.2.3