aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-05-25 19:44:13 +0200
committerDan Engelbrecht <[email protected]>2022-05-25 19:44:13 +0200
commit1014ad5d654e97a2208a435881940de87eb7045e (patch)
tree4ed396cb3d062b4d0200a50c6df43c3394139161
parentcheck out repo to get to CHANGELOG.md (diff)
parentMerge pull request #107 from EpicGames/de/single-worker-clang-verify (diff)
downloadzen-1014ad5d654e97a2208a435881940de87eb7045e.tar.xz
zen-1014ad5d654e97a2208a435881940de87eb7045e.zip
Merge remote-tracking branch 'origin/main' into de/new-release-flow
-rw-r--r--.github/workflows/self_host_build.yml22
-rw-r--r--README.md18
-rw-r--r--zenserver/cache/structuredcache.cpp18
-rw-r--r--zenserver/cache/structuredcachestore.cpp517
-rw-r--r--zenserver/cache/structuredcachestore.h81
-rw-r--r--zenserver/zenserver.cpp4
6 files changed, 471 insertions, 189 deletions
diff --git a/.github/workflows/self_host_build.yml b/.github/workflows/self_host_build.yml
index 3864151ce..8bb57a4b4 100644
--- a/.github/workflows/self_host_build.yml
+++ b/.github/workflows/self_host_build.yml
@@ -9,30 +9,19 @@ jobs:
clang-format:
name: Check clang-format
runs-on: [self-hosted, linux, x64]
- strategy:
- matrix:
- path:
- - 'zen'
- - 'zencore'
- - 'zencore-test'
- - 'zenhttp'
- - 'zenserver-test'
- - 'zenstore'
- - 'zenstore-test'
- - 'zentest-appstub'
- - 'zenutil'
- - 'zenserver'
+
steps:
- uses: actions/checkout@v2
- - name: clang-format ${{ matrix.path }}
+
+ - name: clang-format
uses: jidicula/[email protected]
with:
clang-format-version: '13'
- check-path: ${{ matrix.path }}
+ check-path: '.'
+ exclude-regex: (.*thirdparty.*)
windows-build:
name: Build Windows
- needs: clang-format
runs-on: [self-hosted, windows, x64]
strategy:
matrix:
@@ -81,7 +70,6 @@ jobs:
linux-build:
name: Build Linux
- needs: clang-format
runs-on: [self-hosted, linux, x64]
strategy:
matrix:
diff --git a/README.md b/README.md
index f8cc6779a..0c418324f 100644
--- a/README.md
+++ b/README.md
@@ -194,12 +194,7 @@ xmake build
# Contributing Code
-To run the pre-commit scripts you'll need a few things:
-
-* We rely on clang-format for consistent code formatting. You can install version 12 or later from https://llvm.org/builds/
-* The helper scripts also depend on Python 3.x, which you may install from https://www.python.org/downloads/windows/ (I am presently using 3.9.5 which works). NOTE: *do* check the option to add Python to your PATH!
-
-Once you have those dependencies, you can simply run `prepare_commit.bat` to ensure the code is properly formatted and has the Epic copyright header comment. I'm sure there's a better way to integrate this into the git submit flow but my git-fu is not strong enough yet to know how to do that best.
+See [CODING.md](CODING.md)
# Debugging
@@ -212,7 +207,7 @@ is incredibly handy. When that is installed you may enable auto-attach to child
* `zencore-test` exercises unit tests in the zencore project
* `zenserver-test` exercises the zen server itself (functional tests)
-The tests are implemented using [doctest](https://github.com/onqtam/doctest), which is similar to Catch in usage.
+The tests are implemented using [doctest](https://github.com/onqtam/doctest), which is similar to Catch in usage. We now also support [catch2](https://github.com/catchorg/Catch2)
# Adding a http.sys URL reservation (Windows only)
@@ -223,12 +218,3 @@ Registering a handler for an HTTP endpoint requires either process elevation (i.
or
`netsh http add urlacl url=http://*:1337/ sddl=D:(A;;GX;;;S-1-1-0)` (enable for any authenticated user)
-
-# Coding Standards
-
-See [CODING.md](CODING.md)
-
-Run `prepare_commit.bat` before committing code. It ensures all source files are formatted with
-clang-format which you will need to install.
-
-(More helpful instructions needed here :)
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index e11499289..79d15a204 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -398,7 +398,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
}
void
-HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest& Request, std::string_view)
+HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest& Request, std::string_view Namespace)
{
switch (Request.RequestVerb())
{
@@ -412,14 +412,14 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest&
case HttpVerb::kDelete:
// Drop namespace
{
- // if (m_CacheStore.DropNamespace(Namespace))
- // {
- // return Request.WriteResponse(HttpResponseCode::OK);
- // }
- // else
- // {
- // return Request.WriteResponse(HttpResponseCode::NotFound);
- // }
+ if (m_CacheStore.DropNamespace(Namespace))
+ {
+ return Request.WriteResponse(HttpResponseCode::OK);
+ }
+ else
+ {
+ return Request.WriteResponse(HttpResponseCode::NotFound);
+ }
}
break;
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 7509c5a71..e4dbf390a 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -205,6 +205,36 @@ namespace {
return true;
}
+ bool MoveAndDeleteDirectory(const std::filesystem::path& Dir)
+ {
+ int DropIndex = 0;
+ do
+ {
+ if (!std::filesystem::exists(Dir))
+ {
+ return false;
+ }
+
+ std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex);
+ std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName;
+ if (std::filesystem::exists(DroppedBucketPath))
+ {
+ DropIndex++;
+ continue;
+ }
+
+ std::error_code Ec;
+ std::filesystem::rename(Dir, DroppedBucketPath, Ec);
+ if (!Ec)
+ {
+ DeleteDirectories(DroppedBucketPath);
+ return true;
+ }
+ // TODO: Do we need to bail at some point?
+ zen::Sleep(100);
+ } while (true);
+ }
+
} // namespace
namespace fs = std::filesystem;
@@ -342,6 +372,13 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket)
return AnyDropped;
}
+bool
+ZenCacheNamespace::Drop()
+{
+ m_MemLayer.Drop();
+ return m_DiskLayer.Drop();
+}
+
void
ZenCacheNamespace::Flush()
{
@@ -404,14 +441,14 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa
{
RwLock::SharedLockScope _(m_Lock);
- auto it = m_Buckets.find(std::string(InBucket));
+ auto It = m_Buckets.find(std::string(InBucket));
- if (it == m_Buckets.end())
+ if (It == m_Buckets.end())
{
return false;
}
- CacheBucket* Bucket = &it->second;
+ CacheBucket* Bucket = It->second.get();
_.ReleaseNow();
@@ -425,14 +462,15 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa
void
ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
{
- CacheBucket* Bucket = nullptr;
+ const auto BucketName = std::string(InBucket);
+ CacheBucket* Bucket = nullptr;
{
RwLock::SharedLockScope _(m_Lock);
if (auto It = m_Buckets.find(std::string(InBucket)); It != m_Buckets.end())
{
- Bucket = &It->second;
+ Bucket = It->second.get();
}
}
@@ -444,11 +482,12 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const
if (auto It = m_Buckets.find(std::string(InBucket)); It != m_Buckets.end())
{
- Bucket = &It->second;
+ Bucket = It->second.get();
}
else
{
- Bucket = &m_Buckets[std::string(InBucket)];
+ auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>());
+ Bucket = InsertResult.first->second.get();
}
}
@@ -458,11 +497,37 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const
}
bool
-ZenCacheMemoryLayer::DropBucket(std::string_view Bucket)
+ZenCacheMemoryLayer::DropBucket(std::string_view InBucket)
{
RwLock::ExclusiveLockScope _(m_Lock);
- return !!m_Buckets.erase(std::string(Bucket));
+ auto It = m_Buckets.find(std::string(InBucket));
+
+ if (It != m_Buckets.end())
+ {
+ CacheBucket& Bucket = *It->second;
+ m_DroppedBuckets.push_back(std::move(It->second));
+ m_Buckets.erase(It);
+ Bucket.Drop();
+ return true;
+ }
+ return false;
+}
+
+void
+ZenCacheMemoryLayer::Drop()
+{
+ RwLock::ExclusiveLockScope _(m_Lock);
+ std::vector<std::unique_ptr<CacheBucket>> Buckets;
+ Buckets.reserve(m_Buckets.size());
+ while (!m_Buckets.empty())
+ {
+ const auto& It = m_Buckets.begin();
+ CacheBucket& Bucket = *It->second;
+ m_DroppedBuckets.push_back(std::move(It->second));
+ m_Buckets.erase(It->first);
+ Bucket.Drop();
+ }
}
void
@@ -472,7 +537,7 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
for (auto& Kv : m_Buckets)
{
- Kv.second.Scrub(Ctx);
+ Kv.second->Scrub(Ctx);
}
}
@@ -486,7 +551,7 @@ ZenCacheMemoryLayer::GatherAccessTimes(zen::access_tracking::AccessTimes& Access
for (auto& Kv : m_Buckets)
{
std::vector<KeyAccessTime>& Bucket = AccessTimes.Buckets[Kv.first];
- Kv.second.GatherAccessTimes(Bucket);
+ Kv.second->GatherAccessTimes(Bucket);
}
}
@@ -505,7 +570,7 @@ ZenCacheMemoryLayer::TotalSize() const
for (auto& Kv : m_Buckets)
{
- TotalSize += Kv.second.TotalSize();
+ TotalSize += Kv.second->TotalSize();
}
return TotalSize;
@@ -570,9 +635,16 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue
m_TotalSize.fetch_add(Value.Value.GetSize(), std::memory_order::relaxed);
}
+void
+ZenCacheMemoryLayer::CacheBucket::Drop()
+{
+ RwLock::ExclusiveLockScope _(m_BucketLock);
+ m_CacheMap.clear();
+}
+
//////////////////////////////////////////////////////////////////////////
-ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName))
+ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)), m_BucketId(Oid::Zero)
{
}
@@ -581,19 +653,6 @@ ZenCacheDiskLayer::CacheBucket::~CacheBucket()
}
bool
-ZenCacheDiskLayer::CacheBucket::Delete(std::filesystem::path BucketDir)
-{
- if (std::filesystem::exists(BucketDir))
- {
- DeleteDirectories(BucketDir);
-
- return true;
- }
-
- return false;
-}
-
-void
ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate)
{
using namespace std::literals;
@@ -611,7 +670,10 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
if (Manifest)
{
m_BucketId = Manifest["BucketId"].AsObjectId();
- m_IsOk = m_BucketId != Oid::Zero;
+ if (m_BucketId == Oid::Zero)
+ {
+ return false;
+ }
}
else if (AllowCreate)
{
@@ -625,7 +687,7 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
}
else
{
- return;
+ return false;
}
OpenLog(BucketDir, IsNew);
@@ -641,7 +703,7 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
}
}
- m_IsOk = true;
+ return true;
}
void
@@ -1158,11 +1220,6 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc,
bool
ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue)
{
- if (!m_IsOk)
- {
- return false;
- }
-
RwLock::SharedLockScope _(m_IndexLock);
auto It = m_Index.find(HashKey);
if (It == m_Index.end())
@@ -1184,11 +1241,6 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
void
ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value)
{
- if (!m_IsOk)
- {
- return;
- }
-
if (Value.Value.Size() >= m_LargeObjectThreshold)
{
return PutStandaloneCacheValue(HashKey, Value);
@@ -1196,12 +1248,24 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue&
PutInlineCacheValue(HashKey, Value);
}
-void
+bool
ZenCacheDiskLayer::CacheBucket::Drop()
{
+ RwLock::ExclusiveLockScope _(m_IndexLock);
+
+ std::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks;
+ ShardLocks.reserve(256);
+ for (RwLock& Lock : m_ShardedLocks)
+ {
+ ShardLocks.push_back(std::make_unique<RwLock::ExclusiveLockScope>(Lock));
+ }
m_BlockStore.Close();
m_SlogFile.Close();
- DeleteDirectories(m_BucketDir);
+
+ bool Deleted = MoveAndDeleteDirectory(m_BucketDir);
+
+ m_Index.clear();
+ return Deleted;
}
void
@@ -1703,7 +1767,8 @@ ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx)
for (auto& Kv : m_Buckets)
{
- Kv.second.CollectGarbage(GcCtx);
+ CacheBucket& Bucket = *Kv.second;
+ Bucket.CollectGarbage(GcCtx);
}
}
@@ -1716,7 +1781,7 @@ ZenCacheDiskLayer::UpdateAccessTimes(const zen::access_tracking::AccessTimes& Ac
{
if (auto It = m_Buckets.find(Kv.first); It != m_Buckets.end())
{
- CacheBucket& Bucket = It->second;
+ CacheBucket& Bucket = *It->second;
Bucket.UpdateAccessTimes(Kv.second);
}
}
@@ -1925,11 +1990,11 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
{
RwLock::SharedLockScope _(m_Lock);
- auto it = m_Buckets.find(BucketName);
+ auto It = m_Buckets.find(BucketName);
- if (it != m_Buckets.end())
+ if (It != m_Buckets.end())
{
- Bucket = &it->second;
+ Bucket = It->second.get();
}
}
@@ -1939,24 +2004,27 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
RwLock::ExclusiveLockScope _(m_Lock);
- if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end())
+ if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
{
- Bucket = &it->second;
+ Bucket = It->second.get();
}
else
{
- auto It = m_Buckets.try_emplace(BucketName, BucketName);
- Bucket = &It.first->second;
+ auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName));
+ Bucket = InsertResult.first->second.get();
std::filesystem::path BucketPath = m_RootDir;
BucketPath /= BucketName;
- Bucket->OpenOrCreate(BucketPath);
+ if (!Bucket->OpenOrCreate(BucketPath))
+ {
+ m_Buckets.erase(BucketName);
+ return false;
+ }
}
}
ZEN_ASSERT(Bucket != nullptr);
-
return Bucket->Get(HashKey, OutValue);
}
@@ -1969,11 +2037,11 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
{
RwLock::SharedLockScope _(m_Lock);
- auto it = m_Buckets.find(BucketName);
+ auto It = m_Buckets.find(BucketName);
- if (it != m_Buckets.end())
+ if (It != m_Buckets.end())
{
- Bucket = &it->second;
+ Bucket = It->second.get();
}
}
@@ -1983,28 +2051,29 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
RwLock::ExclusiveLockScope _(m_Lock);
- if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end())
+ if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
{
- Bucket = &it->second;
+ Bucket = It->second.get();
}
else
{
- auto It = m_Buckets.try_emplace(BucketName, BucketName);
- Bucket = &It.first->second;
+ auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName));
+ Bucket = InsertResult.first->second.get();
std::filesystem::path BucketPath = m_RootDir;
BucketPath /= BucketName;
- Bucket->OpenOrCreate(BucketPath);
+ if (!Bucket->OpenOrCreate(BucketPath))
+ {
+ m_Buckets.erase(BucketName);
+ return;
+ }
}
}
ZEN_ASSERT(Bucket != nullptr);
- if (Bucket->IsOk())
- {
- Bucket->Put(HashKey, Value);
- }
+ Bucket->Put(HashKey, Value);
}
void
@@ -2023,26 +2092,20 @@ ZenCacheDiskLayer::DiscoverBuckets()
// New bucket needs to be created
if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
{
+ continue;
}
- else
- {
- auto InsertResult = m_Buckets.try_emplace(BucketName, BucketName);
-
- CacheBucket& Bucket = InsertResult.first->second;
- Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false);
+ auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName));
+ CacheBucket& Bucket = *InsertResult.first->second;
- if (Bucket.IsOk())
- {
- ZEN_INFO("Discovered bucket '{}'", BucketName);
- }
- else
- {
- ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
+ if (!Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false))
+ {
+ ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
- m_Buckets.erase(InsertResult.first);
- }
+ m_Buckets.erase(InsertResult.first);
+ continue;
}
+ ZEN_INFO("Discovered bucket '{}'", BucketName);
}
}
@@ -2051,23 +2114,42 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
{
RwLock::ExclusiveLockScope _(m_Lock);
- auto it = m_Buckets.find(std::string(InBucket));
+ auto It = m_Buckets.find(std::string(InBucket));
- if (it != m_Buckets.end())
+ if (It != m_Buckets.end())
{
- CacheBucket* Bucket = &it->second;
-
- Bucket->Drop();
-
- m_Buckets.erase(it);
+ CacheBucket& Bucket = *It->second;
+ m_DroppedBuckets.push_back(std::move(It->second));
+ m_Buckets.erase(It);
- return true;
+ return Bucket.Drop();
}
+ // Make sure we remove the folder even if we don't know about the bucket
std::filesystem::path BucketPath = m_RootDir;
BucketPath /= std::string(InBucket);
+ return MoveAndDeleteDirectory(BucketPath);
+}
+
+bool
+ZenCacheDiskLayer::Drop()
+{
+ RwLock::ExclusiveLockScope _(m_Lock);
- return CacheBucket::Delete(BucketPath);
+ std::vector<std::unique_ptr<CacheBucket>> Buckets;
+ Buckets.reserve(m_Buckets.size());
+ while (!m_Buckets.empty())
+ {
+ const auto& It = m_Buckets.begin();
+ CacheBucket& Bucket = *It->second;
+ m_DroppedBuckets.push_back(std::move(It->second));
+ m_Buckets.erase(It->first);
+ if (!Bucket.Drop())
+ {
+ return false;
+ }
+ }
+ return MoveAndDeleteDirectory(m_RootDir);
}
void
@@ -2080,7 +2162,8 @@ ZenCacheDiskLayer::Flush()
Buckets.reserve(m_Buckets.size());
for (auto& Kv : m_Buckets)
{
- Buckets.push_back(&Kv.second);
+ CacheBucket* Bucket = Kv.second.get();
+ Buckets.push_back(Bucket);
}
}
@@ -2097,7 +2180,8 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
for (auto& Kv : m_Buckets)
{
- Kv.second.Scrub(Ctx);
+ CacheBucket& Bucket = *Kv.second;
+ Bucket.Scrub(Ctx);
}
}
@@ -2108,7 +2192,8 @@ ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx)
for (auto& Kv : m_Buckets)
{
- Kv.second.GatherReferences(GcCtx);
+ CacheBucket& Bucket = *Kv.second;
+ Bucket.GatherReferences(GcCtx);
}
}
@@ -2120,7 +2205,7 @@ ZenCacheDiskLayer::TotalSize() const
for (auto& Kv : m_Buckets)
{
- TotalSize += Kv.second.TotalSize();
+ TotalSize += Kv.second->TotalSize();
}
return TotalSize;
@@ -2130,12 +2215,16 @@ ZenCacheDiskLayer::TotalSize() const
static constexpr std::string_view UE4DDCNamespaceName = "ue4.ddc";
-ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStorage(Gc), GcContributor(Gc)
+ZenCacheStore::ZenCacheStore(CasGc& Gc, const Configuration& Configuration)
+: GcStorage(Gc)
+, GcContributor(Gc)
+, m_Gc(Gc)
+, m_Configuration(Configuration)
{
- CreateDirectories(BasePath);
+ CreateDirectories(m_Configuration.BasePath);
DirectoryContent DirContent;
- GetDirectoryContent(BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+ GetDirectoryContent(m_Configuration.BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
std::vector<std::string> LegacyBuckets;
std::vector<std::string> Namespaces;
@@ -2150,7 +2239,7 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor
LegacyBuckets.push_back(DirName);
}
- ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size());
+ ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), m_Configuration.BasePath, LegacyBuckets.size());
if (std::find(Namespaces.begin(), Namespaces.end(), UE4DDCNamespaceName) == Namespaces.end())
{
@@ -2158,13 +2247,14 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor
ZEN_INFO("Moving #{} legacy buckets to '{}' namespace", LegacyBuckets.size(), UE4DDCNamespaceName);
- std::filesystem::path DefaultNamespaceFolder = BasePath / fmt::format("{}{}", NamespaceDiskPrefix, UE4DDCNamespaceName);
+ std::filesystem::path DefaultNamespaceFolder =
+ m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, UE4DDCNamespaceName);
CreateDirectories(DefaultNamespaceFolder);
// Move any non-namespace folders into the default namespace folder
for (const std::string& DirName : LegacyBuckets)
{
- std::filesystem::path LegacyFolder = BasePath / DirName;
+ std::filesystem::path LegacyFolder = m_Configuration.BasePath / DirName;
std::filesystem::path NewPath = DefaultNamespaceFolder / DirName;
std::error_code Ec;
std::filesystem::rename(LegacyFolder, NewPath, Ec);
@@ -2179,7 +2269,7 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor
for (const std::string& NamespaceName : Namespaces)
{
m_Namespaces[NamespaceName] =
- std::make_unique<ZenCacheNamespace>(Gc, BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName));
+ std::make_unique<ZenCacheNamespace>(Gc, m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName));
}
}
@@ -2216,7 +2306,22 @@ ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket)
{
return Store->DropBucket(Bucket);
}
- ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}'", Namespace, Bucket);
+ ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropBucket, bucket '{}'", Namespace, Bucket);
+ return false;
+}
+
+bool
+ZenCacheStore::DropNamespace(std::string_view InNamespace)
+{
+ RwLock::SharedLockScope _(m_NamespacesLock);
+ if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end())
+ {
+ ZenCacheNamespace& Namespace = *It->second;
+ m_DroppedNamespaces.push_back(std::move(It->second));
+ m_Namespaces.erase(It);
+ return Namespace.Drop();
+ }
+ ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace);
return false;
}
@@ -2247,13 +2352,29 @@ ZenCacheStore::GetNamespace(std::string_view Namespace)
return It->second.get();
}
}
- return nullptr;
+ _.ReleaseNow();
+
+ if (!m_Configuration.AllowAutomaticCreationOfNamespaces)
+ {
+ return nullptr;
+ }
+
+ RwLock::ExclusiveLockScope __(m_NamespacesLock);
+ if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end())
+ {
+ return It->second.get();
+ }
+
+ auto NewNamespace = m_Namespaces.insert_or_assign(
+ std::string(Namespace),
+ std::make_unique<ZenCacheNamespace>(m_Gc, m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, Namespace)));
+ return NewNamespace.first->second.get();
}
void
ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const
{
- std::vector<std::pair<std::string, ZenCacheNamespace&> > Namespaces;
+ std::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces;
{
RwLock::SharedLockScope _(m_NamespacesLock);
Namespaces.reserve(m_Namespaces.size());
@@ -3048,40 +3169,210 @@ TEST_CASE("z$.namespaces")
ScopedTemporaryDirectory TempDir;
CreateDirectories(TempDir.Path());
+ IoHash Key1;
+ IoHash Key2;
{
CasGc Gc;
- ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+ ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = false});
const auto Bucket = "teardrinker"sv;
const auto CustomNamespace = "mynamespace"sv;
// Create a cache record
- const IoHash Key = CreateKey(42);
- CbObject CacheValue = CreateCacheValue(4096);
+ Key1 = CreateKey(42);
+ CbObject CacheValue = CreateCacheValue(4096);
IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer();
Buffer.SetContentType(ZenContentType::kCbObject);
ZenCacheValue PutValue = {.Value = Buffer};
- Zcs.Put(ZenCacheStore::DefaultNamespace, Bucket, Key, PutValue);
+ Zcs.Put(ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue);
ZenCacheValue GetValue;
- CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key, GetValue));
+ CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue));
+ CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue));
- CHECK(!Zcs.Get(CustomNamespace, Bucket, Key, GetValue));
+ // This should just be dropped as we don't allow creating of namespaces on the fly
+ Zcs.Put(CustomNamespace, Bucket, Key1, PutValue);
+ CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue));
+ }
- // This should just be dropped for now until we decide how we add namespaces
- Zcs.Put(CustomNamespace, Bucket, Key, PutValue);
- CHECK(!Zcs.Get(CustomNamespace, Bucket, Key, GetValue));
+ {
+ CasGc Gc;
+ ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true});
+ const auto Bucket = "teardrinker"sv;
+ const auto CustomNamespace = "mynamespace"sv;
- const IoHash Key2 = CreateKey(43);
- CbObject CacheValue2 = CreateCacheValue(4096);
+ Key2 = CreateKey(43);
+ CbObject CacheValue2 = CreateCacheValue(4096);
IoBuffer Buffer2 = CacheValue2.GetBuffer().AsIoBuffer();
Buffer2.SetContentType(ZenContentType::kCbObject);
ZenCacheValue PutValue2 = {.Value = Buffer2};
Zcs.Put(CustomNamespace, Bucket, Key2, PutValue2);
+ ZenCacheValue GetValue;
CHECK(!Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key2, GetValue));
+ CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue));
+ CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue));
+ CHECK(Zcs.Get(CustomNamespace, Bucket, Key2, GetValue));
+ }
+}
+
+TEST_CASE("z$.drop.bucket")
+{
+ using namespace testutils;
+
+ const auto CreateCacheValue = [](size_t Size) -> CbObject {
+ std::vector<uint8_t> Buf;
+ Buf.resize(Size);
+
+ CbObjectWriter Writer;
+ Writer.AddBinary("Binary"sv, Buf.data(), Buf.size());
+ return Writer.Save();
+ };
+
+ ScopedTemporaryDirectory TempDir;
+ CreateDirectories(TempDir.Path());
+
+ IoHash Key1;
+ IoHash Key2;
+
+ auto PutValue =
+ [&CreateCacheValue](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, size_t KeyIndex, size_t Size) {
+ // Create a cache record
+ IoHash Key = CreateKey(KeyIndex);
+ CbObject CacheValue = CreateCacheValue(Size);
+
+ IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer();
+ Buffer.SetContentType(ZenContentType::kCbObject);
+
+ ZenCacheValue PutValue = {.Value = Buffer};
+ Zcs.Put(Namespace, Bucket, Key, PutValue);
+ return Key;
+ };
+ auto GetValue = [](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) {
+ ZenCacheValue GetValue;
+ Zcs.Get(Namespace, Bucket, Key, GetValue);
+ return GetValue;
+ };
+ WorkerThreadPool Workers(1);
+ {
+ CasGc Gc;
+ ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true});
+ const auto Bucket = "teardrinker"sv;
+ const auto Namespace = "mynamespace"sv;
+
+ Key1 = PutValue(Zcs, Namespace, Bucket, 42, 4096);
+ Key2 = PutValue(Zcs, Namespace, Bucket, 43, 2048);
+
+ ZenCacheValue Value1 = GetValue(Zcs, Namespace, Bucket, Key1);
+ CHECK(Value1.Value);
+
+ std::atomic_bool WorkComplete = false;
+ Workers.ScheduleWork([&]() {
+ zen::Sleep(100);
+ Value1.Value = IoBuffer{};
+ WorkComplete = true;
+ });
+ // On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket
+ // Our DropBucket execution blocks any incoming request from completing until we are done with the drop
+ CHECK(Zcs.DropBucket(Namespace, Bucket));
+ while (!WorkComplete)
+ {
+ zen::Sleep(1);
+ }
+
+ // Entire bucket should be dropped, but doing a request should will re-create the namespace but it must still be empty
+ Value1 = GetValue(Zcs, Namespace, Bucket, Key1);
+ CHECK(!Value1.Value);
+ ZenCacheValue Value2 = GetValue(Zcs, Namespace, Bucket, Key2);
+ CHECK(!Value2.Value);
+ }
+}
+
+TEST_CASE("z$.drop.namespace")
+{
+ using namespace testutils;
+
+ const auto CreateCacheValue = [](size_t Size) -> CbObject {
+ std::vector<uint8_t> Buf;
+ Buf.resize(Size);
+
+ CbObjectWriter Writer;
+ Writer.AddBinary("Binary"sv, Buf.data(), Buf.size());
+ return Writer.Save();
+ };
+
+ ScopedTemporaryDirectory TempDir;
+ CreateDirectories(TempDir.Path());
+
+ auto PutValue =
+ [&CreateCacheValue](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, size_t KeyIndex, size_t Size) {
+ // Create a cache record
+ IoHash Key = CreateKey(KeyIndex);
+ CbObject CacheValue = CreateCacheValue(Size);
+
+ IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer();
+ Buffer.SetContentType(ZenContentType::kCbObject);
+
+ ZenCacheValue PutValue = {.Value = Buffer};
+ Zcs.Put(Namespace, Bucket, Key, PutValue);
+ return Key;
+ };
+ auto GetValue = [](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) {
+ ZenCacheValue GetValue;
+ Zcs.Get(Namespace, Bucket, Key, GetValue);
+ return GetValue;
+ };
+ WorkerThreadPool Workers(1);
+ {
+ CasGc Gc;
+ ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true});
+ const auto Bucket1 = "teardrinker1"sv;
+ const auto Bucket2 = "teardrinker2"sv;
+ const auto Namespace1 = "mynamespace1"sv;
+ const auto Namespace2 = "mynamespace2"sv;
+
+ IoHash Key1 = PutValue(Zcs, Namespace1, Bucket1, 42, 4096);
+ IoHash Key2 = PutValue(Zcs, Namespace1, Bucket2, 43, 2048);
+ IoHash Key3 = PutValue(Zcs, Namespace2, Bucket1, 44, 4096);
+ IoHash Key4 = PutValue(Zcs, Namespace2, Bucket2, 45, 2048);
+
+ ZenCacheValue Value1 = GetValue(Zcs, Namespace1, Bucket1, Key1);
+ CHECK(Value1.Value);
+ ZenCacheValue Value2 = GetValue(Zcs, Namespace1, Bucket2, Key2);
+ CHECK(Value2.Value);
+ ZenCacheValue Value3 = GetValue(Zcs, Namespace2, Bucket1, Key3);
+ CHECK(Value3.Value);
+ ZenCacheValue Value4 = GetValue(Zcs, Namespace2, Bucket2, Key4);
+ CHECK(Value4.Value);
+
+ std::atomic_bool WorkComplete = false;
+ Workers.ScheduleWork([&]() {
+ zen::Sleep(100);
+ Value1.Value = IoBuffer{};
+ Value2.Value = IoBuffer{};
+ Value3.Value = IoBuffer{};
+ Value4.Value = IoBuffer{};
+ WorkComplete = true;
+ });
+ // On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket
+ // Our DropBucket execution blocks any incoming request from completing until we are done with the drop
+ CHECK(Zcs.DropNamespace(Namespace1));
+ while (!WorkComplete)
+ {
+ zen::Sleep(1);
+ }
+
+ // Entire namespace should be dropped, but doing a request should will re-create the namespace but it must still be empty
+ Value1 = GetValue(Zcs, Namespace1, Bucket1, Key1);
+ CHECK(!Value1.Value);
+ Value2 = GetValue(Zcs, Namespace1, Bucket2, Key2);
+ CHECK(!Value2.Value);
+ Value3 = GetValue(Zcs, Namespace2, Bucket1, Key3);
+ CHECK(Value3.Value);
+ Value4 = GetValue(Zcs, Namespace2, Bucket2, Key4);
+ CHECK(Value4.Value);
}
}
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 34b8d18f4..c226074fe 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -150,6 +150,7 @@ public:
bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ void Drop();
bool DropBucket(std::string_view Bucket);
void Scrub(ScrubContext& Ctx);
void GatherAccessTimes(zen::access_tracking::AccessTimes& AccessTimes);
@@ -193,14 +194,16 @@ private:
bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(const IoHash& HashKey, const ZenCacheValue& Value);
+ void Drop();
void Scrub(ScrubContext& Ctx);
void GatherAccessTimes(std::vector<zen::access_tracking::KeyAccessTime>& AccessTimes);
inline uint64_t TotalSize() const { return m_TotalSize; }
};
- mutable RwLock m_Lock;
- std::unordered_map<std::string, CacheBucket> m_Buckets;
- Configuration m_Configuration;
+ mutable RwLock m_Lock;
+ std::unordered_map<std::string, std::unique_ptr<CacheBucket>> m_Buckets;
+ std::vector<std::unique_ptr<CacheBucket>> m_DroppedBuckets;
+ Configuration m_Configuration;
ZenCacheMemoryLayer(const ZenCacheMemoryLayer&) = delete;
ZenCacheMemoryLayer& operator=(const ZenCacheMemoryLayer&) = delete;
@@ -214,6 +217,7 @@ public:
bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ bool Drop();
bool DropBucket(std::string_view Bucket);
void Flush();
void Scrub(ScrubContext& Ctx);
@@ -233,19 +237,16 @@ private:
CacheBucket(std::string BucketName);
~CacheBucket();
- void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
- static bool Delete(std::filesystem::path BucketDir);
- bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(const IoHash& HashKey, const ZenCacheValue& Value);
- void Drop();
- void Flush();
- void SaveManifest();
- void Scrub(ScrubContext& Ctx);
- void GatherReferences(GcContext& GcCtx);
- void CollectGarbage(GcContext& GcCtx);
- void UpdateAccessTimes(const std::vector<zen::access_tracking::KeyAccessTime>& AccessTimes);
+ bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
+ bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(const IoHash& HashKey, const ZenCacheValue& Value);
+ bool Drop();
+ void Flush();
+ void Scrub(ScrubContext& Ctx);
+ void GatherReferences(GcContext& GcCtx);
+ void CollectGarbage(GcContext& GcCtx);
+ void UpdateAccessTimes(const std::vector<zen::access_tracking::KeyAccessTime>& AccessTimes);
- inline bool IsOk() const { return m_IsOk; }
inline uint64_t TotalSize() const { return m_TotalSize.load(std::memory_order::relaxed); }
private:
@@ -257,7 +258,6 @@ private:
std::filesystem::path m_BlocksBasePath;
BlockStore m_BlockStore;
Oid m_BucketId;
- bool m_IsOk = false;
uint64_t m_LargeObjectThreshold = 64 * 1024;
// These files are used to manage storage of small objects for this bucket
@@ -292,16 +292,18 @@ private:
std::atomic_uint64_t m_TotalSize{};
- void BuildPath(PathBuilderBase& Path, const IoHash& HashKey);
- void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
- bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue);
- void PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
- bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue);
- void MakeIndexSnapshot();
- uint64_t ReadIndexFile();
- uint64_t ReadLog(uint64_t LogPosition);
- uint64_t MigrateLegacyData(bool CleanSource);
- void OpenLog(const std::filesystem::path& BucketDir, const bool IsNew);
+ void BuildPath(PathBuilderBase& Path, const IoHash& HashKey);
+ void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+ bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue);
+ void PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+ bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue);
+ void MakeIndexSnapshot();
+ uint64_t ReadIndexFile();
+ uint64_t ReadLog(uint64_t LogPosition);
+ uint64_t MigrateLegacyData(bool CleanSource);
+ void OpenLog(const std::filesystem::path& BucketDir, const bool IsNew);
+ static bool Delete(std::filesystem::path BucketDir);
+ void SaveManifest();
// These locks are here to avoid contention on file creation, therefore it's sufficient
// that we take the same lock for the same hash
@@ -314,9 +316,10 @@ private:
inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
};
- std::filesystem::path m_RootDir;
- mutable RwLock m_Lock;
- std::unordered_map<std::string, CacheBucket> m_Buckets; // TODO: make this case insensitive
+ std::filesystem::path m_RootDir;
+ mutable RwLock m_Lock;
+ std::unordered_map<std::string, std::unique_ptr<CacheBucket>> m_Buckets; // TODO: make this case insensitive
+ std::vector<std::unique_ptr<CacheBucket>> m_DroppedBuckets;
ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete;
ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
@@ -330,6 +333,7 @@ public:
bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ bool Drop();
bool DropBucket(std::string_view Bucket);
void Flush();
void Scrub(ScrubContext& Ctx);
@@ -360,12 +364,19 @@ public:
"!default!"; // This is intentionally not a valid namespace name and will only be used for mapping when no namespace is given
static constexpr std::string_view NamespaceDiskPrefix = "ns_";
- ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath);
+ struct Configuration
+ {
+ std::filesystem::path BasePath;
+ bool AllowAutomaticCreationOfNamespaces = true;
+ };
+
+ ZenCacheStore(CasGc& Gc, const Configuration& Configuration);
~ZenCacheStore();
bool Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
bool DropBucket(std::string_view Namespace, std::string_view Bucket);
+ bool DropNamespace(std::string_view Namespace);
void Flush();
void Scrub(ScrubContext& Ctx);
@@ -377,10 +388,14 @@ private:
ZenCacheNamespace* GetNamespace(std::string_view Namespace);
void IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const;
- typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NameSpaceMap;
+ typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NamespaceMap;
+
+ mutable RwLock m_NamespacesLock;
+ NamespaceMap m_Namespaces;
+ std::vector<std::unique_ptr<ZenCacheNamespace>> m_DroppedNamespaces;
- mutable RwLock m_NamespacesLock;
- NameSpaceMap m_Namespaces;
+ CasGc& m_Gc;
+ Configuration m_Configuration;
};
void z$_forcelink();
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 05ae4a09a..a924d9c81 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -754,7 +754,9 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
using namespace std::literals;
ZEN_INFO("instantiating structured cache service");
- m_CacheStore = std::make_unique<ZenCacheStore>(m_CasGc, m_DataRoot / "cache");
+ m_CacheStore = std::make_unique<ZenCacheStore>(
+ m_CasGc,
+ ZenCacheStore::Configuration{.BasePath = m_DataRoot / "cache", .AllowAutomaticCreationOfNamespaces = true});
const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig;