diff options
| author | Stefan Boberg <[email protected]> | 2021-09-20 12:08:44 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2021-09-20 12:08:44 +0200 |
| commit | 782351959f697fca6b0804c134467b7d9c29da1a (patch) | |
| tree | fdd6c841a567e69e3d0b45089de70c7e7d0bb756 /zenserver/cache | |
| parent | trivial: include cleanup (diff) | |
| download | zen-782351959f697fca6b0804c134467b7d9c29da1a.tar.xz zen-782351959f697fca6b0804c134467b7d9c29da1a.zip | |
Moved more code into zen namespace, for consistency
Also removed snapshot_manifest (remnants of vfs prototype)
Diffstat (limited to 'zenserver/cache')
| -rw-r--r-- | zenserver/cache/structuredcache.cpp | 132 | ||||
| -rw-r--r-- | zenserver/cache/structuredcache.h | 5 | ||||
| -rw-r--r-- | zenserver/cache/structuredcachestore.cpp | 93 | ||||
| -rw-r--r-- | zenserver/cache/structuredcachestore.h | 40 |
4 files changed, 136 insertions, 134 deletions
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index c66b1f98d..b3867bbc3 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -67,7 +67,7 @@ enum class CachePolicy : uint8_t gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy); CachePolicy -ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) +ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams) { CachePolicy QueryPolicy = CachePolicy::Query; @@ -76,7 +76,7 @@ ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) if (!Opts.empty()) { QueryPolicy = CachePolicy::None; - zen::ForEachStrTok(Opts, ',', [&QueryPolicy](const std::string_view& Opt) { + ForEachStrTok(Opts, ',', [&QueryPolicy](const std::string_view& Opt) { if (Opt == detail::cacheopt::Local) { QueryPolicy |= CachePolicy::QueryLocal; @@ -97,7 +97,7 @@ ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) if (!Opts.empty()) { StorePolicy = CachePolicy::None; - zen::ForEachStrTok(Opts, ',', [&StorePolicy](const std::string_view& Opt) { + ForEachStrTok(Opts, ',', [&StorePolicy](const std::string_view& Opt) { if (Opt == detail::cacheopt::Local) { StorePolicy |= CachePolicy::StoreLocal; @@ -117,7 +117,7 @@ ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) std::string_view Opts = QueryParams.GetValue("skip"sv); if (!Opts.empty()) { - zen::ForEachStrTok(Opts, ',', [&SkipPolicy](const std::string_view& Opt) { + ForEachStrTok(Opts, ',', [&SkipPolicy](const std::string_view& Opt) { if (Opt == detail::cacheopt::Meta) { SkipPolicy |= CachePolicy::SkipMeta; @@ -144,11 +144,11 @@ ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) ////////////////////////////////////////////////////////////////////////// -HttpStructuredCacheService::HttpStructuredCacheService(::ZenCacheStore& InCacheStore, - zen::CasStore& InStore, - zen::CidStore& InCidStore, +HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, + CasStore& InStore, + CidStore& InCidStore, std::unique_ptr<UpstreamCache> UpstreamCache) -: m_Log(zen::logging::Get("cache")) +: m_Log(logging::Get("cache")) , m_CacheStore(InCacheStore) , m_CasStore(InStore) , m_CidStore(InCidStore) @@ -173,7 +173,7 @@ HttpStructuredCacheService::Flush() } void -HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) +HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { CacheRef Ref; @@ -188,7 +188,7 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) return HandleCacheBucketRequest(Request, Key); } - return Request.WriteResponse(zen::HttpResponseCode::BadRequest); // invalid URL + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL } const auto QueryParams = Request.GetQueryParams(); @@ -207,12 +207,12 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) { ZEN_UNUSED(Request, Bucket); switch (auto Verb = Request.RequestVerb()) { - using enum zen::HttpVerb; + using enum HttpVerb; case kHead: case kGet: @@ -226,22 +226,22 @@ HttpStructuredCacheService::HandleCacheBucketRequest(zen::HttpServerRequest& Req if (m_CacheStore.DropBucket(Bucket)) { - return Request.WriteResponse(zen::HttpResponseCode::OK); + return Request.WriteResponse(HttpResponseCode::OK); } else { - return Request.WriteResponse(zen::HttpResponseCode::NotFound); + return Request.WriteResponse(HttpResponseCode::NotFound); } break; } } void -HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) +HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) { switch (auto Verb = Request.RequestVerb()) { - using enum zen::HttpVerb; + using enum HttpVerb; case kHead: case kGet: @@ -253,7 +253,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req bool InUpstreamCache = false; const bool QueryUpstream = - !Success && m_UpstreamCache && (zen::CachePolicy::QueryRemote == (Policy & zen::CachePolicy::QueryRemote)); + !Success && m_UpstreamCache && (CachePolicy::QueryRemote == (Policy & CachePolicy::QueryRemote)); if (QueryUpstream) { @@ -272,14 +272,14 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { if (CacheRecordType == ZenContentType::kCbObject) { - const zen::CbValidateError ValidationResult = - zen::ValidateCompactBinary(UpstreamResult.Value, zen::CbValidateMode::All); + const CbValidateError ValidationResult = + ValidateCompactBinary(UpstreamResult.Value, CbValidateMode::All); if (ValidationResult == CbValidateError::None) { - zen::CbObjectView CacheRecord(UpstreamResult.Value.Data()); + CbObjectView CacheRecord(UpstreamResult.Value.Data()); - zen::CbObjectWriter IndexData; + CbObjectWriter IndexData; IndexData.BeginArray("references"); CacheRecord.IterateAttachments([&](CbFieldView Attachment) { IndexData.AddHash(Attachment.AsHash()); }); IndexData.EndArray(); @@ -355,7 +355,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { ZEN_DEBUG("MISS - '{}/{}'", Ref.BucketSegment, Ref.HashKey); - return Request.WriteResponse(zen::HttpResponseCode::NotFound); + return Request.WriteResponse(HttpResponseCode::NotFound); } if (Verb == kHead) @@ -367,13 +367,13 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { CbObjectView CacheRecord(Value.Value.Data()); - const zen::CbValidateError ValidationResult = zen::ValidateCompactBinary(Value.Value, zen::CbValidateMode::All); + const CbValidateError ValidationResult = ValidateCompactBinary(Value.Value, CbValidateMode::All); if (ValidationResult != CbValidateError::None) { ZEN_WARN("GET - cache record '{}/{}' FAILED, invalid compact binary object", Ref.BucketSegment, Ref.HashKey); - return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv); } uint32_t AttachmentCount = 0; @@ -401,7 +401,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req FoundCount, AttachmentCount); - return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv); } Package.SetObject(LoadCompactBinaryObject(Value.Value)); @@ -419,7 +419,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req IoBuffer Response(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); - return Request.WriteResponse(zen::HttpResponseCode::OK, HttpContentType::kCbPackage, Response); + return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, Response); } else { @@ -429,23 +429,23 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req NiceBytes(Value.Value.Size()), InUpstreamCache ? "UPSTREAM" : "LOCAL"); - return Request.WriteResponse(zen::HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); + return Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); } } break; case kPut: { - zen::IoBuffer Body = Request.ReadPayload(); + IoBuffer Body = Request.ReadPayload(); if (!Body || Body.Size() == 0) { - return Request.WriteResponse(zen::HttpResponseCode::BadRequest); + return Request.WriteResponse(HttpResponseCode::BadRequest); } const HttpContentType ContentType = Request.RequestContentType(); - const bool StoreUpstream = m_UpstreamCache && (zen::CachePolicy::StoreRemote == (Policy & zen::CachePolicy::StoreRemote)); + const bool StoreUpstream = m_UpstreamCache && (CachePolicy::StoreRemote == (Policy & CachePolicy::StoreRemote)); if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kUnknownContentType) { @@ -459,13 +459,13 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req {.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}}); } - return Request.WriteResponse(zen::HttpResponseCode::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else if (ContentType == HttpContentType::kCbObject) { // Validate payload before accessing it - const zen::CbValidateError ValidationResult = - zen::ValidateCompactBinary(MemoryView(Body.Data(), Body.Size()), zen::CbValidateMode::All); + const CbValidateError ValidationResult = + ValidateCompactBinary(MemoryView(Body.Data(), Body.Size()), CbValidateMode::All); if (ValidationResult != CbValidateError::None) { @@ -481,7 +481,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req } // Extract referenced payload hashes - zen::CbObjectView Cbo(Body.Data()); + CbObjectView Cbo(Body.Data()); std::vector<IoHash> References; std::vector<IoHash> MissingRefs; @@ -492,7 +492,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req if (!References.empty()) { - zen::CbObjectWriter Idx; + CbObjectWriter Idx; Idx.BeginArray("references"); for (const IoHash& Hash : References) @@ -514,7 +514,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req ZEN_DEBUG("PUT - cache record '{}/{}' {}, {}/{} attachments missing", Ref.BucketSegment, Ref.HashKey, - zen::NiceBytes(CacheValue.Value.Size()), + NiceBytes(CacheValue.Value.Size()), MissingRefs.size(), References.size()); @@ -525,12 +525,12 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req .CacheKey = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(References)}); - return Request.WriteResponse(zen::HttpResponseCode::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else { // TODO: Binary attachments? - zen::CbObjectWriter Response; + CbObjectWriter Response; Response.BeginArray("needs"); for (const IoHash& MissingRef : MissingRefs) { @@ -540,7 +540,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req Response.EndArray(); // Return Created | BadRequest? - return Request.WriteResponse(zen::HttpResponseCode::Created, Response.Save()); + return Request.WriteResponse(HttpResponseCode::Created, Response.Save()); } } else if (ContentType == HttpContentType::kCbPackage) @@ -631,17 +631,17 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req ZEN_DEBUG("PUT - cache record '{}/{}' {}, {}/{} ({}/{}) new attachments", Ref.BucketSegment, Ref.HashKey, - zen::NiceBytes(TotalPackageBytes), + NiceBytes(TotalPackageBytes), AttachmentResult.NewCount, AttachmentResult.Count, - zen::NiceBytes(AttachmentResult.NewBytes), - zen::NiceBytes(AttachmentResult.Bytes)); + NiceBytes(AttachmentResult.NewBytes), + NiceBytes(AttachmentResult.Bytes)); - return Request.WriteResponse(zen::HttpResponseCode::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else { - return Request.WriteResponse(zen::HttpResponseCode::BadRequest); + return Request.WriteResponse(HttpResponseCode::BadRequest); } } break; @@ -655,7 +655,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req } void -HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) +HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) { // Note: the URL references the uncompressed payload hash - so this maintains the mapping // from uncompressed CAS identity (aka CID/Content ID) to the stored payload hash @@ -667,12 +667,12 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re switch (auto Verb = Request.RequestVerb()) { - using enum zen::HttpVerb; + using enum HttpVerb; case kHead: case kGet: { - zen::IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId); + IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId); bool InUpstreamCache = false; if (!Payload && m_UpstreamCache) @@ -680,11 +680,11 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (auto UpstreamResult = m_UpstreamCache->GetCachePayload({{Ref.BucketSegment, Ref.HashKey}, Ref.PayloadId}); UpstreamResult.Success) { - if (zen::CompressedBuffer Compressed = zen::CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) { Payload = UpstreamResult.Value; - zen::IoHash ChunkHash = zen::IoHash::HashBuffer(Payload); - zen::CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash); + IoHash ChunkHash = IoHash::HashBuffer(Payload); + CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash); InUpstreamCache = true; m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); @@ -699,7 +699,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (!Payload) { ZEN_DEBUG("MISS - '{}/{}/{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId); - return Request.WriteResponse(zen::HttpResponseCode::NotFound); + return Request.WriteResponse(HttpResponseCode::NotFound); } ZEN_DEBUG("HIT - '{}/{}/{}' {} (type: {}) ({})", @@ -715,29 +715,29 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re Request.SetSuppressResponseBody(); } - return Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Payload); + return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload); } break; case kPut: { - if (zen::IoBuffer Body = Request.ReadPayload()) + if (IoBuffer Body = Request.ReadPayload()) { if (Body.Size() == 0) { - return Request.WriteResponse(zen::HttpResponseCode::BadRequest, + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Empty payload not permitted"); } - zen::IoHash ChunkHash = zen::IoHash::HashBuffer(Body); + IoHash ChunkHash = IoHash::HashBuffer(Body); - zen::CompressedBuffer Compressed = zen::CompressedBuffer::FromCompressed(SharedBuffer(Body)); + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Body)); if (!Compressed) { // All attachment payloads need to be in compressed buffer format - return Request.WriteResponse(zen::HttpResponseCode::BadRequest, + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Attachments must be compressed"); } @@ -749,7 +749,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re return Request.WriteResponse(HttpResponseCode::BadRequest); } - zen::CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash); + CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash); m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); @@ -763,11 +763,11 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (Result.New) { - return Request.WriteResponse(zen::HttpResponseCode::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else { - return Request.WriteResponse(zen::HttpResponseCode::OK); + return Request.WriteResponse(HttpResponseCode::OK); } } } @@ -783,7 +783,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re } bool -HttpStructuredCacheService::ValidateKeyUri(zen::HttpServerRequest& Request, CacheRef& OutRef) +HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& OutRef) { std::string_view Key = Request.RelativeUri(); std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); @@ -819,14 +819,14 @@ HttpStructuredCacheService::ValidateKeyUri(zen::HttpServerRequest& Request, Cach PayloadSegment = Key.substr(PayloadSplitOffset + 1); } - if (HashSegment.size() != zen::IoHash::StringLength) + if (HashSegment.size() != IoHash::StringLength) { return false; } - if (!PayloadSegment.empty() && PayloadSegment.size() == zen::IoHash::StringLength) + if (!PayloadSegment.empty() && PayloadSegment.size() == IoHash::StringLength) { - const bool IsOk = zen::ParseHexBytes(PayloadSegment.data(), PayloadSegment.size(), OutRef.PayloadId.Hash); + const bool IsOk = ParseHexBytes(PayloadSegment.data(), PayloadSegment.size(), OutRef.PayloadId.Hash); if (!IsOk) { @@ -835,10 +835,10 @@ HttpStructuredCacheService::ValidateKeyUri(zen::HttpServerRequest& Request, Cach } else { - OutRef.PayloadId = zen::IoHash::Zero; + OutRef.PayloadId = IoHash::Zero; } - const bool IsOk = zen::ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); + const bool IsOk = ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); if (!IsOk) { diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index 796b21d1f..8871094b3 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -10,13 +10,12 @@ namespace spdlog { class logger; } -class ZenCacheStore; - namespace zen { class CasStore; class CidStore; class UpstreamCache; +class ZenCacheStore; enum class CachePolicy : uint8_t; /** @@ -77,7 +76,7 @@ private: spdlog::logger& Log() { return m_Log; } spdlog::logger& m_Log; - ZenCacheStore& m_CacheStore; + zen::ZenCacheStore& m_CacheStore; zen::CasStore& m_CasStore; zen::CidStore& m_CidStore; std::unique_ptr<UpstreamCache> m_UpstreamCache; diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 018955e65..14f4531ab 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -24,15 +24,16 @@ #include <atlfile.h> -using namespace zen; -using namespace fmt::literals; - ////////////////////////////////////////////////////////////////////////// -ZenCacheStore::ZenCacheStore(zen::CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir} +namespace zen { + +using namespace fmt::literals; + +ZenCacheStore::ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir} { ZEN_INFO("initializing structured cache at '{}'", RootDir); - zen::CreateDirectories(RootDir); + CreateDirectories(RootDir); } ZenCacheStore::~ZenCacheStore() @@ -40,7 +41,7 @@ ZenCacheStore::~ZenCacheStore() } bool -ZenCacheStore::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue) +ZenCacheStore::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue) { bool Ok = m_MemLayer.Get(InBucket, HashKey, OutValue); @@ -68,7 +69,7 @@ ZenCacheStore::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCac } void -ZenCacheStore::Put(std::string_view InBucket, const zen::IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) { // Store value and index @@ -115,7 +116,7 @@ ZenCacheMemoryLayer::~ZenCacheMemoryLayer() } bool -ZenCacheMemoryLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue) +ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue) { CacheBucket* Bucket = nullptr; @@ -139,7 +140,7 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, } void -ZenCacheMemoryLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) { CacheBucket* Bucket = nullptr; @@ -179,7 +180,7 @@ ZenCacheMemoryLayer::DropBucket(std::string_view Bucket) } bool -ZenCacheMemoryLayer::CacheBucket::Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue) +ZenCacheMemoryLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue) { RwLock::SharedLockScope _(m_bucketLock); @@ -196,7 +197,7 @@ ZenCacheMemoryLayer::CacheBucket::Get(const zen::IoHash& HashKey, ZenCacheValue& } void -ZenCacheMemoryLayer::CacheBucket::Put(const zen::IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value) { RwLock::ExclusiveLockScope _(m_bucketLock); @@ -227,7 +228,7 @@ struct DiskLocation struct DiskIndexEntry { - zen::IoHash Key; + IoHash Key; DiskLocation Location; }; @@ -243,8 +244,8 @@ struct ZenCacheDiskLayer::CacheBucket void OpenOrCreate(std::filesystem::path BucketDir); static bool Delete(std::filesystem::path BucketDir); - bool Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue); - void Put(const zen::IoHash& HashKey, const ZenCacheValue& Value); + bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(const IoHash& HashKey, const ZenCacheValue& Value); void Drop(); void Flush(); @@ -260,12 +261,12 @@ private: BasicFile m_SobsFile; TCasLogFile<DiskIndexEntry> m_SlogFile; - void BuildPath(zen::WideStringBuilderBase& Path, const zen::IoHash& HashKey); - void PutLargeObject(const zen::IoHash& HashKey, const ZenCacheValue& Value); + void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey); + void PutLargeObject(const IoHash& HashKey, const ZenCacheValue& Value); - RwLock m_IndexLock; - tsl::robin_map<zen::IoHash, DiskLocation, zen::IoHash::Hasher> m_Index; - uint64_t m_WriteCursor = 0; + RwLock m_IndexLock; + tsl::robin_map<IoHash, DiskLocation, IoHash::Hasher> m_Index; + uint64_t m_WriteCursor = 0; }; ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas) @@ -281,7 +282,7 @@ ZenCacheDiskLayer::CacheBucket::Delete(std::filesystem::path BucketDir) { if (std::filesystem::exists(BucketDir)) { - zen::DeleteDirectories(BucketDir); + DeleteDirectories(BucketDir); return true; } @@ -292,7 +293,7 @@ ZenCacheDiskLayer::CacheBucket::Delete(std::filesystem::path BucketDir) void ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir) { - zen::CreateDirectories(BucketDir); + CreateDirectories(BucketDir); m_BucketDir = BucketDir; @@ -357,7 +358,7 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir) uint64_t MaxFileOffset = 0; - if (zen::RwLock::ExclusiveLockScope _(m_IndexLock); m_Index.empty()) + if (RwLock::ExclusiveLockScope _(m_IndexLock); m_Index.empty()) { m_SlogFile.Replay([&](const DiskIndexEntry& Record) { m_Index[Record.Key] = Record.Location; @@ -372,7 +373,7 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir) } void -ZenCacheDiskLayer::CacheBucket::BuildPath(zen::WideStringBuilderBase& Path, const zen::IoHash& HashKey) +ZenCacheDiskLayer::CacheBucket::BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey) { char hex[sizeof(HashKey.Hash) * 2]; ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, hex); @@ -383,14 +384,14 @@ ZenCacheDiskLayer::CacheBucket::BuildPath(zen::WideStringBuilderBase& Path, cons } bool -ZenCacheDiskLayer::CacheBucket::Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue) +ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue) { if (!m_Ok) { return false; } - zen::RwLock::SharedLockScope _(m_IndexLock); + RwLock::SharedLockScope _(m_IndexLock); if (auto it = m_Index.find(HashKey); it != m_Index.end()) { @@ -417,7 +418,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const zen::IoHash& HashKey, ZenCacheValue& O WideStringBuilder<128> DataFilePath; BuildPath(DataFilePath, HashKey); - if (zen::IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str())) + if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str())) { OutValue.Value = Data; OutValue.Value.SetContentType(ContentType); @@ -431,7 +432,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const zen::IoHash& HashKey, ZenCacheValue& O } void -ZenCacheDiskLayer::CacheBucket::Put(const zen::IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value) { if (!m_Ok) { @@ -453,12 +454,12 @@ ZenCacheDiskLayer::CacheBucket::Put(const zen::IoHash& HashKey, const ZenCacheVa EntryFlags |= DiskLocation::kStructured; } - zen::RwLock::ExclusiveLockScope _(m_IndexLock); + RwLock::ExclusiveLockScope _(m_IndexLock); DiskLocation Loc{.OffsetAndFlags = DiskLocation::CombineOffsetAndFlags(m_WriteCursor, EntryFlags), .Size = gsl::narrow<uint32_t>(Value.Value.Size())}; - m_WriteCursor = zen::RoundUp(m_WriteCursor + Loc.Size, 16); + m_WriteCursor = RoundUp(m_WriteCursor + Loc.Size, 16); if (auto it = m_Index.find(HashKey); it == m_Index.end()) { @@ -483,7 +484,7 @@ ZenCacheDiskLayer::CacheBucket::Drop() m_SobsFile.Close(); m_SlogFile.Close(); - zen::DeleteDirectories(m_BucketDir); + DeleteDirectories(m_BucketDir); } void @@ -494,9 +495,9 @@ ZenCacheDiskLayer::CacheBucket::Flush() } void -ZenCacheDiskLayer::CacheBucket::PutLargeObject(const zen::IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheDiskLayer::CacheBucket::PutLargeObject(const IoHash& HashKey, const ZenCacheValue& Value) { - zen::WideStringBuilder<128> DataFilePath; + WideStringBuilder<128> DataFilePath; BuildPath(DataFilePath, HashKey); // TODO: replace this with a more efficient implementation with proper atomic rename @@ -507,21 +508,21 @@ ZenCacheDiskLayer::CacheBucket::PutLargeObject(const zen::IoHash& HashKey, const if (FAILED(hRes)) { - zen::ThrowSystemException(hRes, "Failed to open temporary file for put at '{}'"_format(m_BucketDir)); + ThrowSystemException(hRes, "Failed to open temporary file for put at '{}'"_format(m_BucketDir)); } hRes = DataFile.Write(Value.Value.Data(), gsl::narrow<DWORD>(Value.Value.Size())); if (FAILED(hRes)) { - zen::ThrowSystemException(hRes, "Failed to write payload ({} bytes) to file"_format(NiceBytes(Value.Value.Size()))); + ThrowSystemException(hRes, "Failed to write payload ({} bytes) to file"_format(NiceBytes(Value.Value.Size()))); } hRes = DataFile.Close(DataFilePath.c_str()); if (FAILED(hRes)) { - zen::ThrowSystemException(hRes, "Failed to finalize file '{}'"_format(zen::WideToUtf8(DataFilePath))); + ThrowSystemException(hRes, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath))); } // Update index @@ -533,7 +534,7 @@ ZenCacheDiskLayer::CacheBucket::PutLargeObject(const zen::IoHash& HashKey, const EntryFlags |= DiskLocation::kStructured; } - zen::RwLock::ExclusiveLockScope _(m_IndexLock); + RwLock::ExclusiveLockScope _(m_IndexLock); DiskLocation Loc{.OffsetAndFlags = DiskLocation::CombineOffsetAndFlags(0, EntryFlags), .Size = 0}; @@ -560,12 +561,12 @@ ZenCacheDiskLayer::ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& ZenCacheDiskLayer::~ZenCacheDiskLayer() = default; bool -ZenCacheDiskLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue) +ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue) { CacheBucket* Bucket = nullptr; { - zen::RwLock::SharedLockScope _(m_Lock); + RwLock::SharedLockScope _(m_Lock); auto it = m_Buckets.find(std::string(InBucket)); @@ -579,7 +580,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, Ze { // Bucket needs to be opened/created - zen::RwLock::ExclusiveLockScope _(m_Lock); + RwLock::ExclusiveLockScope _(m_Lock); if (auto it = m_Buckets.find(std::string(InBucket)); it != m_Buckets.end()) { @@ -603,12 +604,12 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, Ze } void -ZenCacheDiskLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, const ZenCacheValue& Value) +ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) { CacheBucket* Bucket = nullptr; { - zen::RwLock::SharedLockScope _(m_Lock); + RwLock::SharedLockScope _(m_Lock); auto it = m_Buckets.find(std::string(InBucket)); @@ -622,7 +623,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, co { // New bucket needs to be created - zen::RwLock::ExclusiveLockScope _(m_Lock); + RwLock::ExclusiveLockScope _(m_Lock); if (auto it = m_Buckets.find(std::string(InBucket)); it != m_Buckets.end()) { @@ -651,7 +652,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, co bool ZenCacheDiskLayer::DropBucket(std::string_view InBucket) { - zen::RwLock::ExclusiveLockScope _(m_Lock); + RwLock::ExclusiveLockScope _(m_Lock); auto it = m_Buckets.find(std::string(InBucket)); @@ -679,7 +680,7 @@ ZenCacheDiskLayer::Flush() Buckets.reserve(m_Buckets.size()); { - zen::RwLock::SharedLockScope _(m_Lock); + RwLock::SharedLockScope _(m_Lock); for (auto& Kv : m_Buckets) { @@ -705,7 +706,7 @@ ZenCacheTracker::~ZenCacheTracker() } void -ZenCacheTracker::TrackAccess(std::string_view Bucket, const zen::IoHash& HashKey) +ZenCacheTracker::TrackAccess(std::string_view Bucket, const IoHash& HashKey) { ZEN_UNUSED(Bucket); ZEN_UNUSED(HashKey); @@ -715,3 +716,5 @@ void ZenCacheTracker::Flush() { } + +} // namespace zen diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 48c3cfde9..c4fa20958 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -23,8 +23,6 @@ namespace zen { class WideStringBuilderBase; class CasStore; -} // namespace zen - /****************************************************************************** /$$$$$$$$ /$$$$$$ /$$ @@ -44,8 +42,8 @@ class CasStore; struct ZenCacheValue { - zen::IoBuffer Value; - zen::CbObject IndexData; + IoBuffer Value; + CbObject IndexData; }; class ZenCacheMemoryLayer @@ -54,32 +52,32 @@ public: ZenCacheMemoryLayer(); ~ZenCacheMemoryLayer(); - bool Get(std::string_view Bucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue); - void Put(std::string_view Bucket, const zen::IoHash& HashKey, const ZenCacheValue& Value); + bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); bool DropBucket(std::string_view Bucket); private: struct CacheBucket { - zen::RwLock m_bucketLock; - tsl::robin_map<zen::IoHash, zen::IoBuffer> m_cacheMap; + RwLock m_bucketLock; + tsl::robin_map<IoHash, IoBuffer> m_cacheMap; - bool Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue); - void Put(const zen::IoHash& HashKey, const ZenCacheValue& Value); + bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(const IoHash& HashKey, const ZenCacheValue& Value); }; - zen::RwLock m_Lock; + RwLock m_Lock; std::unordered_map<std::string, CacheBucket> m_Buckets; }; class ZenCacheDiskLayer { public: - ZenCacheDiskLayer(zen::CasStore& Cas, const std::filesystem::path& RootDir); + ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir); ~ZenCacheDiskLayer(); - bool Get(std::string_view Bucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue); - void Put(std::string_view Bucket, const zen::IoHash& HashKey, const ZenCacheValue& Value); + bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); bool DropBucket(std::string_view Bucket); void Flush(); @@ -89,20 +87,20 @@ private: */ struct CacheBucket; - zen::CasStore& m_CasStore; + CasStore& m_CasStore; std::filesystem::path m_RootDir; - zen::RwLock m_Lock; + RwLock m_Lock; std::unordered_map<std::string, CacheBucket> m_Buckets; // TODO: make this case insensitive }; class ZenCacheStore { public: - ZenCacheStore(zen::CasStore& Cas, const std::filesystem::path& RootDir); + ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir); ~ZenCacheStore(); - bool Get(std::string_view Bucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue); - void Put(std::string_view Bucket, const zen::IoHash& HashKey, const ZenCacheValue& Value); + bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); bool DropBucket(std::string_view Bucket); void Flush(); @@ -121,8 +119,10 @@ public: ZenCacheTracker(ZenCacheStore& CacheStore); ~ZenCacheTracker(); - void TrackAccess(std::string_view Bucket, const zen::IoHash& HashKey); + void TrackAccess(std::string_view Bucket, const IoHash& HashKey); void Flush(); private: }; + +} // namespace zen |