aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--zencore/filesystem.cpp2
-rw-r--r--zencore/include/zencore/sharedbuffer.h6
-rw-r--r--zencore/memory.cpp4
-rw-r--r--zenserver-test/zenserver-test.cpp17
-rw-r--r--zenserver/cache/cacheagent.cpp5
-rw-r--r--zenserver/cache/cacheagent.h9
-rw-r--r--zenserver/cache/structuredcachestore.cpp128
-rw-r--r--zenserver/cache/structuredcachestore.h7
-rw-r--r--zenserver/zenserver.cpp28
-rw-r--r--zenserver/zenserver.vcxproj2
-rw-r--r--zenserver/zenserver.vcxproj.filters6
11 files changed, 166 insertions, 48 deletions
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index b8e8eff55..a642e2cf6 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -840,7 +840,7 @@ TEST_CASE("filesystem")
// GetExePath -- this is not a great test as it's so dependent on where the this code gets linked in
path BinPath = GetRunningExecutablePath();
- const bool ExpectedExe = BinPath.stem() == "zencore-test" || BinPath.stem() == "zenserver-test" || BinPath.stem() == "zenstore-test";
+ const bool ExpectedExe = ToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver";
CHECK(ExpectedExe);
CHECK(is_regular_file(BinPath));
diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h
index 640c3fe74..1f87dc639 100644
--- a/zencore/include/zencore/sharedbuffer.h
+++ b/zencore/include/zencore/sharedbuffer.h
@@ -143,6 +143,12 @@ public:
/** Make a non-owned view of the input */
[[nodiscard]] inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); }
+ /** Make a non-owning view of the memory of the contiguous container. */
+ [[nodiscard]] inline static SharedBuffer MakeView(const std::ranges::contiguous_range auto& Container)
+ {
+ std::span Span = Container;
+ return MakeView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type));
+ }
/** Make a non-owned view of the input */
[[nodiscard]] ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size);
/** Make a non-owned view of the input */
diff --git a/zencore/memory.cpp b/zencore/memory.cpp
index 62c81076d..14ea7ca1d 100644
--- a/zencore/memory.cpp
+++ b/zencore/memory.cpp
@@ -185,13 +185,13 @@ TEST_CASE("ChunkingLinearAllocator")
TEST_CASE("MemoryView")
{
{
- uint8_t Array1[16];
+ uint8_t Array1[16] = {};
MemoryView View1 = MakeMemoryView(Array1);
CHECK(View1.GetSize() == 16);
}
{
- uint32_t Array2[16];
+ uint32_t Array2[16] = {};
MemoryView View2 = MakeMemoryView(Array2);
CHECK(View2.GetSize() == 64);
}
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 23be9f729..f3849b2cd 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -66,6 +66,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
#endif
using namespace fmt::literals;
+using namespace std::literals;
/*
@@ -126,10 +127,10 @@ public:
m_Headers.reserve(16);
zen::ExtendableStringBuilder<256> RequestBody;
- RequestBody << "GET " << Path << " HTTP/1.1\r\n";
- RequestBody << "Host: " << Server << "\r\n";
- RequestBody << "Accept: */*\r\n";
- RequestBody << "Connection: " << (m_KeepAlive ? "keep-alive" : "close") << "\r\n\r\n"; // TODO: support keep-alive
+ RequestBody << "GET "sv << Path << " HTTP/1.1\r\n"sv;
+ RequestBody << "Host: "sv << Server << "\r\n"sv;
+ RequestBody << "Accept: */*\r\n"sv;
+ RequestBody << "Connection: "sv << (m_KeepAlive ? "keep-alive"sv : "close"sv) << "\r\n\r\n"sv; // TODO: support keep-alive
m_RequestBody = RequestBody;
@@ -460,12 +461,12 @@ using namespace spdlog;
using namespace spdlog::details;
using namespace std::literals;
-class full_formatter final : public spdlog::formatter
+class full_test_formatter final : public spdlog::formatter
{
public:
- full_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch) : m_Epoch(Epoch), m_LogId(LogId) {}
+ full_test_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch) : m_Epoch(Epoch), m_LogId(LogId) {}
- virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_formatter>(m_LogId, m_Epoch); }
+ virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_test_formatter>(m_LogId, m_Epoch); }
static constexpr bool UseDate = false;
@@ -680,7 +681,7 @@ main(int argc, char** argv)
zen::logging::InitializeLogging();
spdlog::set_level(spdlog::level::debug);
- spdlog::set_formatter(std::make_unique<::logging::full_formatter>("test", std::chrono::system_clock::now()));
+ spdlog::set_formatter(std::make_unique<::logging::full_test_formatter>("test", std::chrono::system_clock::now()));
std::filesystem::path ProgramBaseDir = std::filesystem::path(argv[0]).parent_path();
std::filesystem::path TestBaseDir = ProgramBaseDir.parent_path().parent_path() / ".test";
diff --git a/zenserver/cache/cacheagent.cpp b/zenserver/cache/cacheagent.cpp
deleted file mode 100644
index f4d1cabe6..000000000
--- a/zenserver/cache/cacheagent.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "cacheagent.h"
-
-#include <gsl/gsl-lite.hpp>
diff --git a/zenserver/cache/cacheagent.h b/zenserver/cache/cacheagent.h
deleted file mode 100644
index 145d0f79f..000000000
--- a/zenserver/cache/cacheagent.h
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-class CacheAgent
-{
-public:
-private:
-};
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index e7b840ed8..02ec43115 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -6,19 +6,27 @@
#include <zencore/windows.h>
#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/compress.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/iobuffer.h>
#include <zencore/logging.h>
#include <zencore/string.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
#include <zencore/thread.h>
#include <zenstore/basicfile.h>
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
+#include <zenstore/cidstore.h>
#include <concepts>
#include <filesystem>
#include <unordered_map>
+#include <ranges>
ZEN_THIRD_PARTY_INCLUDES_START
#include <fmt/core.h>
@@ -31,7 +39,7 @@ namespace zen {
using namespace fmt::literals;
-ZenCacheStore::ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir}
+ZenCacheStore::ZenCacheStore(const std::filesystem::path& RootDir) : m_DiskLayer{RootDir}
{
ZEN_INFO("initializing structured cache at '{}'", RootDir);
CreateDirectories(RootDir);
@@ -125,7 +133,8 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
void
ZenCacheStore::GarbageCollect(GcContext& GcCtx)
{
- ZEN_UNUSED(GcCtx);
+ m_DiskLayer.GarbageCollect(GcCtx);
+ m_MemLayer.GarbageCollect(GcCtx);
}
//////////////////////////////////////////////////////////////////////////
@@ -213,7 +222,12 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
void
ZenCacheMemoryLayer::GarbageCollect(GcContext& GcCtx)
{
- ZEN_UNUSED(GcCtx);
+ RwLock::SharedLockScope _(m_Lock);
+
+ for (auto& Kv : m_Buckets)
+ {
+ Kv.second.GarbageCollect(GcCtx);
+ }
}
void
@@ -358,7 +372,7 @@ static_assert(sizeof(DiskIndexEntry) == 36);
struct ZenCacheDiskLayer::CacheBucket
{
- CacheBucket(CasStore& Cas);
+ CacheBucket();
~CacheBucket();
void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
@@ -374,7 +388,6 @@ struct ZenCacheDiskLayer::CacheBucket
inline bool IsOk() const { return m_IsOk; }
private:
- CasStore& m_CasStore;
std::filesystem::path m_BucketDir;
Oid m_BucketId;
bool m_IsOk = false;
@@ -405,7 +418,7 @@ private:
inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
};
-ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas)
+ZenCacheDiskLayer::CacheBucket::CacheBucket()
{
}
@@ -839,7 +852,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
//////////////////////////////////////////////////////////////////////////
-ZenCacheDiskLayer::ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir) : m_RootDir(RootDir), m_CasStore(Cas)
+ZenCacheDiskLayer::ZenCacheDiskLayer(const std::filesystem::path& RootDir) : m_RootDir(RootDir)
{
}
@@ -873,7 +886,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
}
else
{
- auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
+ auto It = m_Buckets.try_emplace(std::string(InBucket));
Bucket = &It.first->second;
std::filesystem::path BucketPath = m_RootDir;
@@ -916,7 +929,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
}
else
{
- auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
+ auto It = m_Buckets.try_emplace(std::string(InBucket));
Bucket = &It.first->second;
std::filesystem::path bucketPath = m_RootDir;
@@ -972,7 +985,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
}
else
{
- auto InsertResult = m_Buckets.try_emplace(BucketName8, m_CasStore);
+ auto InsertResult = m_Buckets.try_emplace(BucketName8);
std::filesystem::path BucketPath = m_RootDir;
BucketPath /= BucketName8;
@@ -1050,7 +1063,100 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
void
ZenCacheDiskLayer::GarbageCollect(GcContext& GcCtx)
{
- ZEN_UNUSED(GcCtx);
+ RwLock::SharedLockScope _(m_Lock);
+
+ for (auto& Kv : m_Buckets)
+ {
+ Kv.second.GarbageCollect(GcCtx);
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("z$.store")
+{
+ using namespace fmt::literals;
+ using namespace std::literals;
+
+ ScopedTemporaryDirectory TempDir;
+
+ ZenCacheStore Zcs(TempDir.Path() / "cache");
+
+ const int kIterationCount = 100;
+
+ for (int i = 0; i < kIterationCount; ++i)
+ {
+ const IoHash Key = IoHash::HashBuffer(&i, sizeof i);
+
+ CbObjectWriter Cbo;
+ Cbo << "hey" << i;
+ CbObject Obj = Cbo.Save();
+
+ ZenCacheValue Value;
+ Value.Value = Obj.GetBuffer().AsIoBuffer();
+ Value.Value.SetContentType(ZenContentType::kCbObject);
+
+ Zcs.Put("test_bucket"sv, Key, Value);
+ }
+
+ for (int i = 0; i < kIterationCount; ++i)
+ {
+ const IoHash Key = IoHash::HashBuffer(&i, sizeof i);
+
+ ZenCacheValue Value;
+ Zcs.Get("test_bucket"sv, Key, /* out */ Value);
+
+ REQUIRE(Value.Value);
+ CHECK(Value.Value.GetContentType() == ZenContentType::kCbObject);
+ CHECK_EQ(ValidateCompactBinary(Value.Value, CbValidateMode::All), CbValidateError::None);
+ CbObject Obj = LoadCompactBinaryObject(Value.Value);
+ CHECK_EQ(Obj["hey"].AsInt32(), i);
+ }
+}
+
+TEST_CASE("z$.gc")
+{
+ using namespace fmt::literals;
+ using namespace std::literals;
+
+ ScopedTemporaryDirectory TempDir;
+
+ CasStoreConfiguration Config{.RootDirectory = TempDir.Path()};
+ std::unique_ptr<CasStore> Cas{CreateCasStore()};
+ Cas->Initialize(Config);
+
+ CidStore Cid(*Cas, TempDir.Path());
+
+ ZenCacheStore Zcs(TempDir.Path() / "cache");
+
+ const int kIterationCount = 100;
+
+ for (int i = 0; i < kIterationCount; ++i)
+ {
+ const IoHash Key = IoHash::HashBuffer(&i, sizeof i);
+
+ CompressedBuffer CompBuf = CompressedBuffer::Compress(SharedBuffer::MakeView("abcd"sv));
+ Cid.AddChunk(CompBuf);
+
+ CbObjectWriter Cbo;
+ Cbo << "hey" << i << "ref" << CbAttachment(CompBuf);
+ CbObject Obj = Cbo.Save();
+
+ ZenCacheValue Value;
+ Value.Value = Obj.GetBuffer().AsIoBuffer();
+ Value.Value.SetContentType(ZenContentType::kCbObject);
+
+ Zcs.Put("test_bucket"sv, Key, Value);
+ }
+}
+
+#endif
+
+void
+z$_forcelink()
+{
}
} // namespace zen
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 4753af627..760f4995c 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -101,7 +101,7 @@ private:
class ZenCacheDiskLayer
{
public:
- ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir);
+ explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir);
~ZenCacheDiskLayer();
bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -119,7 +119,6 @@ private:
*/
struct CacheBucket;
- CasStore& m_CasStore;
std::filesystem::path m_RootDir;
RwLock m_Lock;
std::unordered_map<std::string, CacheBucket> m_Buckets; // TODO: make this case insensitive
@@ -128,7 +127,7 @@ private:
class ZenCacheStore
{
public:
- ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir);
+ explicit ZenCacheStore(const std::filesystem::path& RootDir);
~ZenCacheStore();
bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -146,4 +145,6 @@ private:
uint64_t m_LastScrubTime = 0;
};
+void z$_forcelink();
+
} // namespace zen
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 80bc6a6d4..9a8090fc0 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -627,7 +627,7 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
auto ValueOrDefault = [](std::string_view Value, std::string_view Default) { return Value.empty() ? Default : Value; };
ZEN_INFO("instantiating structured cache service");
- m_CacheStore = std::make_unique<ZenCacheStore>(*m_CasStore, m_DataRoot / "cache");
+ m_CacheStore = std::make_unique<ZenCacheStore>(m_DataRoot / "cache");
std::unique_ptr<zen::UpstreamCache> UpstreamCache;
if (ServiceConfig.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled)
@@ -890,6 +890,22 @@ ZenWindowsService::Run()
return 0;
}
+#if ZEN_WITH_TESTS
+int
+test_main(int argc, char** argv)
+{
+ zen::zencore_forcelinktests();
+ zen::zenhttp_forcelinktests();
+ zen::zenstore_forcelinktests();
+
+ zen::logging::InitializeLogging();
+
+ spdlog::set_level(spdlog::level::debug);
+
+ return doctest::Context(argc, argv).run();
+}
+#endif
+
int
main(int argc, char* argv[])
{
@@ -899,6 +915,16 @@ main(int argc, char* argv[])
mi_version();
#endif
+#if ZEN_WITH_TESTS
+ if (argc >= 2)
+ {
+ if (argv[1] == "test"sv)
+ {
+ return test_main(argc, argv);
+ }
+ }
+#endif
+
try
{
ZenServerOptions GlobalOptions;
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index b670582e7..d954d3f8d 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -120,7 +120,6 @@
<ClInclude Include="testing\httptest.h" />
<ClInclude Include="upstream\jupiter.h" />
<ClInclude Include="projectstore.h" />
- <ClInclude Include="cache\cacheagent.h" />
<ClInclude Include="testing\launch.h" />
<ClInclude Include="casstore.h" />
<ClInclude Include="diag\diagsvcs.h" />
@@ -142,7 +141,6 @@
<ClCompile Include="monitoring\httpstats.cpp" />
<ClCompile Include="monitoring\httpstatus.cpp" />
<ClCompile Include="projectstore.cpp" />
- <ClCompile Include="cache\cacheagent.cpp" />
<ClCompile Include="sos\sos.cpp" />
<ClCompile Include="testing\httptest.cpp" />
<ClCompile Include="upstream\jupiter.cpp" />
diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters
index b87fa0016..04c6267ba 100644
--- a/zenserver/zenserver.vcxproj.filters
+++ b/zenserver/zenserver.vcxproj.filters
@@ -5,9 +5,6 @@
<ClInclude Include="projectstore.h" />
<ClInclude Include="casstore.h" />
<ClInclude Include="testing\launch.h" />
- <ClInclude Include="cache\cacheagent.h">
- <Filter>cache</Filter>
- </ClInclude>
<ClInclude Include="diag\diagsvcs.h">
<Filter>diag</Filter>
</ClInclude>
@@ -49,9 +46,6 @@
<ClCompile Include="zenserver.cpp" />
<ClCompile Include="projectstore.cpp" />
<ClCompile Include="casstore.cpp" />
- <ClCompile Include="cache\cacheagent.cpp">
- <Filter>cache</Filter>
- </ClCompile>
<ClCompile Include="experimental\usnjournal.cpp">
<Filter>experimental</Filter>
</ClCompile>
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-28 07:04:00 +0000