aboutsummaryrefslogtreecommitdiff
path: root/zenserver
diff options
context:
space:
mode:
Diffstat (limited to 'zenserver')
-rw-r--r--zenserver/cache/structuredcache.cpp15
-rw-r--r--zenserver/cache/structuredcache.h2
-rw-r--r--zenserver/cache/structuredcachestore.cpp57
-rw-r--r--zenserver/cache/structuredcachestore.h34
-rw-r--r--zenserver/diag/logging.cpp4
-rw-r--r--zenserver/projectstore.h7
-rw-r--r--zenserver/upstream/zen.h6
-rw-r--r--zenserver/zenserver.cpp46
8 files changed, 104 insertions, 67 deletions
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 35cb02cbb..ffb6f563a 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -148,7 +148,6 @@ ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams)
//////////////////////////////////////////////////////////////////////////
HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- CasStore& InStore,
CidStore& InCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
@@ -157,7 +156,6 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCac
, m_CacheStore(InCacheStore)
, m_StatsService(StatsService)
, m_StatusService(StatusService)
-, m_CasStore(InStore)
, m_CidStore(InCidStore)
, m_UpstreamCache(std::move(UpstreamCache))
{
@@ -194,7 +192,6 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx)
m_LastScrubTime = Ctx.ScrubTimestamp();
- m_CasStore.Scrub(Ctx);
m_CidStore.Scrub(Ctx);
m_CacheStore.Scrub(Ctx);
}
@@ -716,12 +713,8 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
{
if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value)))
{
- Payload = UpstreamResult.Value;
- IoHash ChunkHash = IoHash::HashBuffer(Payload);
- CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash);
- InUpstreamCache = true;
-
- m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
+ InUpstreamCache = true;
}
else
{
@@ -789,9 +782,7 @@ HttpStructuredCacheService::HandlePutCachePayload(zen::HttpServerRequest& Reques
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Payload ID does not match attachment hash"sv);
}
- CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash);
-
- m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({})",
Ref.BucketSegment,
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index ad7253f79..68a01becb 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -54,7 +54,6 @@ class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider
{
public:
HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- CasStore& InCasStore,
CidStore& InCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
@@ -98,7 +97,6 @@ private:
ZenCacheStore& m_CacheStore;
HttpStatsService& m_StatsService;
HttpStatusService& m_StatusService;
- CasStore& m_CasStore;
CidStore& m_CidStore;
std::unique_ptr<UpstreamCache> m_UpstreamCache;
uint64_t m_LastScrubTime = 0;
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index e1d77a088..b120f3955 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -26,6 +26,7 @@
#include <concepts>
#include <filesystem>
+#include <memory_resource>
#include <ranges>
#include <unordered_map>
@@ -40,7 +41,7 @@ namespace zen {
using namespace fmt::literals;
-ZenCacheStore::ZenCacheStore(const std::filesystem::path& RootDir) : m_DiskLayer{RootDir}
+ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) : GcContributor(Gc), m_DiskLayer{RootDir}
{
ZEN_INFO("initializing structured cache at '{}'", RootDir);
CreateDirectories(RootDir);
@@ -89,6 +90,25 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa
m_DiskLayer.Put(InBucket, HashKey, Value);
+#if ZEN_USE_REF_TRACKING
+ if (Value.Value.GetContentType() == ZenContentType::kCbObject)
+ {
+ if (ValidateCompactBinary(Value.Value, CbValidateMode::All) == CbValidateError::None)
+ {
+ CbObject Object{SharedBuffer(Value.Value)};
+
+ uint8_t TempBuffer[8 * sizeof(IoHash)];
+ std::pmr::monotonic_buffer_resource Linear{TempBuffer, sizeof TempBuffer};
+ std::pmr::polymorphic_allocator Allocator{&Linear};
+ std::pmr::vector<IoHash> CidReferences{Allocator};
+
+ Object.IterateAttachments([&](CbFieldView Field) { CidReferences.push_back(Field.AsAttachment()); });
+
+ m_Gc.OnNewCidReferences(CidReferences);
+ }
+ }
+#endif
+
if (Value.Value.Size() <= m_DiskLayerSizeThreshold)
{
m_MemLayer.Put(InBucket, HashKey, Value);
@@ -132,10 +152,10 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheStore::GarbageCollect(GcContext& GcCtx)
+ZenCacheStore::GatherReferences(GcContext& GcCtx)
{
- m_DiskLayer.GarbageCollect(GcCtx);
- m_MemLayer.GarbageCollect(GcCtx);
+ m_MemLayer.GatherReferences(GcCtx);
+ m_DiskLayer.GatherReferences(GcCtx);
}
//////////////////////////////////////////////////////////////////////////
@@ -221,13 +241,13 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheMemoryLayer::GarbageCollect(GcContext& GcCtx)
+ZenCacheMemoryLayer::GatherReferences(GcContext& GcCtx)
{
RwLock::SharedLockScope _(m_Lock);
for (auto& Kv : m_Buckets)
{
- Kv.second.GarbageCollect(GcCtx);
+ Kv.second.GatherReferences(GcCtx);
}
}
@@ -253,7 +273,7 @@ ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheMemoryLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
+ZenCacheMemoryLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
{
// Is it even meaningful to do this? The memory layer shouldn't
// contain anything which is not already in the disk layer
@@ -378,13 +398,12 @@ struct ZenCacheDiskLayer::CacheBucket
void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
static bool Delete(std::filesystem::path BucketDir);
-
- bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(const IoHash& HashKey, const ZenCacheValue& Value);
- void Drop();
- void Flush();
- void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(const IoHash& HashKey, const ZenCacheValue& Value);
+ void Drop();
+ void Flush();
+ void Scrub(ScrubContext& Ctx);
+ void GatherReferences(GcContext& GcCtx);
inline bool IsOk() const { return m_IsOk; }
@@ -716,7 +735,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheDiskLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
+ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
{
RwLock::SharedLockScope _(m_IndexLock);
@@ -1062,13 +1081,13 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheDiskLayer::GarbageCollect(GcContext& GcCtx)
+ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx)
{
RwLock::SharedLockScope _(m_Lock);
for (auto& Kv : m_Buckets)
{
- Kv.second.GarbageCollect(GcCtx);
+ Kv.second.GatherReferences(GcCtx);
}
}
@@ -1083,7 +1102,9 @@ TEST_CASE("z$.store")
ScopedTemporaryDirectory TempDir;
- ZenCacheStore Zcs(TempDir.Path() / "cache");
+ CasGc Gc;
+
+ ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
const int kIterationCount = 100;
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 760f4995c..6beecf78b 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -8,11 +8,11 @@
#include <zencore/thread.h>
#include <zencore/uid.h>
#include <zenstore/cas.h>
+#include <zenstore/gc.h>
-#pragma warning(push)
-#pragma warning(disable : 4127)
+ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
-#pragma warning(pop)
+ZEN_THIRD_PARTY_INCLUDES_END
#include <compare>
#include <filesystem>
@@ -22,6 +22,7 @@ namespace zen {
class WideStringBuilderBase;
class CasStore;
+class CasGc;
/******************************************************************************
@@ -50,6 +51,9 @@ struct ZenCacheValue
Intended for small values which are frequently accessed
+ This should have a better memory management policy to maintain reasonable
+ footprint.
+
*/
class ZenCacheMemoryLayer
{
@@ -61,7 +65,7 @@ public:
void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
bool DropBucket(std::string_view Bucket);
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
struct Configuration
{
@@ -87,7 +91,7 @@ private:
bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(const IoHash& HashKey, const ZenCacheValue& Value);
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
private:
uint64_t GetCurrentTimeStamp();
@@ -109,7 +113,7 @@ public:
bool DropBucket(std::string_view Bucket);
void Flush();
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
void DiscoverBuckets();
@@ -124,24 +128,24 @@ private:
std::unordered_map<std::string, CacheBucket> m_Buckets; // TODO: make this case insensitive
};
-class ZenCacheStore
+class ZenCacheStore : public GcContributor
{
public:
- explicit ZenCacheStore(const std::filesystem::path& RootDir);
+ ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir);
~ZenCacheStore();
- bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
- bool DropBucket(std::string_view Bucket);
- void Flush();
- void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ bool DropBucket(std::string_view Bucket);
+ void Flush();
+ void Scrub(ScrubContext& Ctx);
+ virtual void GatherReferences(GcContext& GcCtx) override;
private:
std::filesystem::path m_RootDir;
ZenCacheMemoryLayer m_MemLayer;
ZenCacheDiskLayer m_DiskLayer;
- uint64_t m_DiskLayerSizeThreshold = 4 * 1024;
+ uint64_t m_DiskLayerSizeThreshold = 1 * 1024;
uint64_t m_LastScrubTime = 0;
};
diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp
index 6e2559f1f..74c5c2101 100644
--- a/zenserver/diag/logging.cpp
+++ b/zenserver/diag/logging.cpp
@@ -272,12 +272,12 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
auto HttpLogger = std::make_shared<spdlog::logger>("http_requests", HttpSink);
spdlog::register_logger(HttpLogger);
- // Jupiter - only log HTTP traffic to file
+ // Jupiter - only log upstream HTTP traffic to file
auto JupiterLogger = std::make_shared<spdlog::logger>("jupiter", FileSink);
spdlog::register_logger(JupiterLogger);
- // Zen - only log HTTP traffic to file
+ // Zen - only log upstream HTTP traffic to file
auto ZenClientLogger = std::make_shared<spdlog::logger>("zenclient", FileSink);
spdlog::register_logger(ZenClientLogger);
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index c9f49217a..83d3986bb 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -2,6 +2,7 @@
#pragma once
+#include <zencore/logging.h>
#include <zencore/uid.h>
#include <zencore/xxhash.h>
#include <zenhttp/httpserver.h>
@@ -9,13 +10,15 @@
#include <zenstore/caslog.h>
#include <zenstore/cidstore.h>
-#include <tsl/robin_map.h>
-#include <zencore/logging.h>
#include <filesystem>
#include <map>
#include <optional>
#include <string>
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
namespace zen {
class CbPackage;
diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h
index 12e46bd8d..7f55294ce 100644
--- a/zenserver/upstream/zen.h
+++ b/zenserver/upstream/zen.h
@@ -10,12 +10,10 @@
#include <zencore/uid.h>
#include <zencore/zencore.h>
-#pragma warning(push)
-#pragma warning(disable : 4127)
+ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
-#pragma warning(pop)
-
#include <asio.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
#include <chrono>
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 9a8090fc0..33cec7db1 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -19,6 +19,9 @@
#include <zenstore/cidstore.h>
#include <zenutil/zenserverprocess.h>
+#define ZEN_USE_NAMED_PIPES 0
+#define ZEN_USE_EXEC 0
+
#if ZEN_USE_MIMALLOC
ZEN_THIRD_PARTY_INCLUDES_START
# include <mimalloc-new-delete.h>
@@ -43,7 +46,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
# define BUILD_VERSION ("dev-build")
#endif
-#define ZEN_SCHEMA_VERSION 1
+#define ZEN_SCHEMA_VERSION 2 /* latest change by: stefan boberg */
//////////////////////////////////////////////////////////////////////////
// We don't have any doctest code in this file but this is needed to bring
@@ -217,13 +220,18 @@ public:
m_ProjectStore = new zen::ProjectStore(*m_CidStore, m_DataRoot / "projects");
m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore});
+
+#if ZEN_USE_NAMED_PIPES
m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore);
+#endif
ZEN_INFO("instantiating compute services");
+#if ZEN_USE_EXEC
std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox";
zen::CreateDirectories(SandboxDir);
m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir);
+#endif
std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply";
zen::CreateDirectories(ApplySandboxDir);
@@ -265,10 +273,12 @@ public:
m_Http->RegisterService(*m_StructuredCacheService);
}
+#if ZEN_USE_EXEC
if (m_HttpLaunchService)
{
m_Http->RegisterService(*m_HttpLaunchService);
}
+#endif
if (m_HttpFunctionService)
{
@@ -430,6 +440,16 @@ public:
NiceByteRate(Ctx.ScrubbedBytes(), ElapsedTimeMs));
}
+ void CollectGarbage()
+ {
+ Stopwatch Timer;
+ ZEN_INFO("Garbage collection STARTING");
+
+ m_Gc.CollectGarbage();
+
+ ZEN_INFO("Garbage collection DONE after {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ }
+
void Flush()
{
if (m_CasStore)
@@ -493,17 +513,15 @@ private:
zen::Ref<zen::HttpServer> m_Http;
zen::HttpStatusService m_StatusService;
zen::HttpStatsService m_StatsService;
- std::unique_ptr<zen::CasStore> m_CasStore{zen::CreateCasStore()};
+ zen::CasGc m_Gc;
+ std::unique_ptr<zen::CasStore> m_CasStore{zen::CreateCasStore(m_Gc)};
std::unique_ptr<zen::CidStore> m_CidStore;
std::unique_ptr<zen::ZenCacheStore> m_CacheStore;
- zen::CasGc m_Gc{*m_CasStore};
zen::CasScrubber m_Scrubber{*m_CasStore};
zen::HttpTestService m_TestService;
zen::HttpTestingService m_TestingService;
zen::HttpCasService m_CasService{*m_CasStore};
zen::RefPtr<zen::ProjectStore> m_ProjectStore;
- zen::Ref<zen::LocalProjectService> m_LocalProjectService;
- std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService;
std::unique_ptr<zen::HttpProjectService> m_HttpProjectService;
std::unique_ptr<zen::HttpStructuredCacheService> m_StructuredCacheService;
zen::HttpAdminService m_AdminService;
@@ -512,6 +530,14 @@ private:
std::unique_ptr<zen::HttpFunctionService> m_HttpFunctionService;
std::unique_ptr<zen::HttpFrontendService> m_FrontendService;
+#if ZEN_USE_EXEC
+ std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService;
+#endif
+
+#if ZEN_USE_NAMED_PIPES
+ zen::Ref<zen::LocalProjectService> m_LocalProjectService;
+#endif
+
bool m_DebugOptionForcedCrash = false;
};
@@ -627,7 +653,7 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
auto ValueOrDefault = [](std::string_view Value, std::string_view Default) { return Value.empty() ? Default : Value; };
ZEN_INFO("instantiating structured cache service");
- m_CacheStore = std::make_unique<ZenCacheStore>(m_DataRoot / "cache");
+ m_CacheStore = std::make_unique<ZenCacheStore>(m_Gc, m_DataRoot / "cache");
std::unique_ptr<zen::UpstreamCache> UpstreamCache;
if (ServiceConfig.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled)
@@ -723,12 +749,8 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
}
}
- m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(*m_CacheStore,
- *m_CasStore,
- *m_CidStore,
- m_StatsService,
- m_StatusService,
- std::move(UpstreamCache)));
+ m_StructuredCacheService.reset(
+ new zen::HttpStructuredCacheService(*m_CacheStore, *m_CidStore, m_StatsService, m_StatusService, std::move(UpstreamCache)));
}
} // namespace zen
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-30 00:40:14 +0000