aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/projectstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/projectstore')
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.cpp618
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.h5
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp167
-rw-r--r--src/zenserver/projectstore/httpprojectstore.h7
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.cpp4
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.h3
-rw-r--r--src/zenserver/projectstore/projectstore.cpp200
-rw-r--r--src/zenserver/projectstore/projectstore.h13
8 files changed, 597 insertions, 420 deletions
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index ab96ae92d..52cdc5983 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -3,14 +3,12 @@
#include "buildsremoteprojectstore.h"
#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinaryvalidation.h>
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
#include <zenhttp/httpclientauth.h>
-
-#include <zenutil/jupiter/jupiterclient.h>
-#include <zenutil/jupiter/jupitersession.h>
+#include <zenutil/jupiter/jupiterbuildstorage.h>
namespace zen {
@@ -21,20 +19,24 @@ static const std::string_view OplogContainerPartName = "oplogcontainer"sv;
class BuildsRemoteStore : public RemoteProjectStore
{
public:
- BuildsRemoteStore(Ref<JupiterClient>&& InJupiterClient,
- std::string_view Namespace,
- std::string_view Bucket,
- const Oid& BuildId,
- const IoBuffer& MetaData,
- bool ForceDisableBlocks,
- bool ForceDisableTempBlocks,
- const std::filesystem::path& TempFilePath)
- : m_JupiterClient(std::move(InJupiterClient))
+ BuildsRemoteStore(std::unique_ptr<BuildStorage::Statistics>&& BuildStorageStats,
+ std::unique_ptr<HttpClient>&& BuildStorageHttp,
+ std::unique_ptr<BuildStorage>&& BuildStorage,
+ std::string_view Url,
+ std::string_view Namespace,
+ std::string_view Bucket,
+ const Oid& BuildId,
+ const IoBuffer& MetaData,
+ bool ForceDisableBlocks,
+ bool ForceDisableTempBlocks)
+ : m_BuildStorageStats(std::move(BuildStorageStats))
+ , m_BuildStorageHttp(std::move(BuildStorageHttp))
+ , m_BuildStorage(std::move(BuildStorage))
+ , m_Url(Url)
, m_Namespace(Namespace)
, m_Bucket(Bucket)
, m_BuildId(BuildId)
, m_MetaData(MetaData)
- , m_TempFilePath(TempFilePath)
, m_EnableBlocks(!ForceDisableBlocks)
, m_UseTempBlocks(!ForceDisableTempBlocks)
{
@@ -47,63 +49,91 @@ public:
.UseTempBlockFiles = m_UseTempBlocks,
.AllowChunking = true,
.ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId),
- .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId)};
+ .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_Url, m_Namespace, m_Bucket, m_BuildId)};
}
virtual Stats GetStats() const override
{
- return {.m_SentBytes = m_SentBytes.load(),
- .m_ReceivedBytes = m_ReceivedBytes.load(),
- .m_RequestTimeNS = m_RequestTimeNS.load(),
- .m_RequestCount = m_RequestCount.load(),
- .m_PeakSentBytes = m_PeakSentBytes.load(),
- .m_PeakReceivedBytes = m_PeakReceivedBytes.load(),
- .m_PeakBytesPerSec = m_PeakBytesPerSec.load()};
+ return {
+ .m_SentBytes = m_BuildStorageStats->TotalBytesWritten.load(),
+ .m_ReceivedBytes = m_BuildStorageStats->TotalBytesRead.load(),
+ .m_RequestTimeNS = m_BuildStorageStats->TotalRequestTimeUs.load() * 1000,
+ .m_RequestCount = m_BuildStorageStats->TotalRequestCount.load(),
+ .m_PeakSentBytes = m_BuildStorageStats->PeakSentBytes.load(),
+ .m_PeakReceivedBytes = m_BuildStorageStats->PeakReceivedBytes.load(),
+ .m_PeakBytesPerSec = m_BuildStorageStats->PeakBytesPerSec.load(),
+ };
}
virtual CreateContainerResult CreateContainer() override
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
-
- IoBuffer Payload = m_MetaData;
- Payload.SetContentType(ZenContentType::kCbObject);
- JupiterResult PutResult = Session.PutBuild(m_Namespace, m_Bucket, m_BuildId, Payload);
- AddStats(PutResult);
+ CreateContainerResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
- CreateContainerResult Result{ConvertResult(PutResult)};
- if (Result.ErrorCode)
+ CbObject Payload = LoadCompactBinaryObject(m_MetaData);
+ try
{
- Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- Result.Reason);
+ CbObject PutBuildResult = m_BuildStorage->PutBuild(m_BuildId, Payload);
+ ZEN_UNUSED(PutBuildResult);
+ m_OplogBuildPartId = Oid::NewOid();
}
- m_OplogBuildPartId = Oid::NewOid();
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason =
+ fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason =
+ fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+ }
+
return Result;
}
virtual SaveResult SaveContainer(const IoBuffer& Payload) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- PutBuildPartResult PutResult =
- Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload);
- AddStats(PutResult);
- SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash};
- if (Result.ErrorCode)
+ SaveResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
+
+ try
{
- Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- m_OplogBuildPartId,
- Result.Reason);
+ CbObject ObjectPayload = LoadCompactBinaryObject(Payload);
+
+ std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
+ m_BuildStorage->PutBuildPart(m_BuildId, m_OplogBuildPartId, OplogContainerPartName, ObjectPayload);
+ Result.RawHash = PutBuildPartResult.first;
+ Result.Needs = std::unordered_set<IoHash, IoHash::Hasher>(PutBuildPartResult.second.begin(), PutBuildPartResult.second.end());
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ m_OplogBuildPartId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ m_OplogBuildPartId,
+ Ex.what());
}
return Result;
@@ -114,52 +144,84 @@ public:
ChunkBlockDescription&& Block) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- JupiterResult PutResult =
- Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
- AddStats(PutResult);
+ SaveAttachmentResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
- SaveAttachmentResult Result{ConvertResult(PutResult)};
- if (Result.ErrorCode)
+ try
{
- Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- RawHash,
- Result.Reason);
- return Result;
- }
+ m_BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
- if (Block.BlockHash == RawHash)
- {
- CbObjectWriter BlockMetaData;
- BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string());
-
- IoBuffer MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()).GetBuffer().AsIoBuffer();
- MetaPayload.SetContentType(ZenContentType::kCbObject);
- JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, RawHash, MetaPayload);
- AddStats(PutMetaResult);
- RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult);
- if (MetaDataResult.ErrorCode)
+ if (Block.BlockHash == RawHash)
{
- ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- RawHash,
- MetaDataResult.Reason);
+ try
+ {
+ CbObjectWriter BlockMetaData;
+ BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string());
+ CbObject MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save());
+ if (!m_BuildStorage->PutBlockMetadata(m_BuildId, RawHash, MetaPayload))
+ {
+ ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ RawHash,
+ "not found");
+ }
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
}
}
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
+
return Result;
}
virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override
{
SaveAttachmentsResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
for (const SharedBuffer& Chunk : Chunks)
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer());
@@ -177,38 +239,68 @@ public:
ZEN_UNUSED(RawHash);
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- FinalizeBuildPartResult FinalizeRefResult =
- Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash);
- AddStats(FinalizeRefResult);
+ FinalizeResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
- FinalizeResult Result{ConvertResult(FinalizeRefResult), {FinalizeRefResult.Needs.begin(), FinalizeRefResult.Needs.end()}};
- if (Result.ErrorCode)
+ try
+ {
+ std::vector<IoHash> Needs = m_BuildStorage->FinalizeBuildPart(m_BuildId, m_OplogBuildPartId, RawHash);
+ Result.Needs = std::unordered_set<IoHash, IoHash::Hasher>(Needs.begin(), Needs.end());
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0;
+ Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ m_OplogBuildPartId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
{
- Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- m_OplogBuildPartId,
- Result.Reason);
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ m_OplogBuildPartId,
+ Ex.what());
}
- else if (Result.Needs.empty())
+
+ if (!Result.ErrorCode && Result.Needs.empty())
{
- JupiterResult FinalizeBuildResult = Session.FinalizeBuild(m_Namespace, m_Bucket, m_BuildId);
- AddStats(FinalizeBuildResult);
- FinalizeBuildResult.ElapsedSeconds += FinalizeRefResult.ElapsedSeconds;
- Result = {ConvertResult(FinalizeBuildResult)};
- if (Result.ErrorCode)
+ try
{
- Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- FinalizeBuildResult.Reason);
+ m_BuildStorage->FinalizeBuild(m_BuildId);
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = Ex.m_Error != 0 ? Ex.m_Error
+ : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode
+ : 0;
+ Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
+ }
+ catch (const std::exception& Ex)
+ {
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ Ex.what());
}
}
+
return Result;
}
@@ -216,161 +308,128 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- JupiterResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId);
- AddStats(GetBuildResult);
- LoadContainerResult Result{ConvertResult(GetBuildResult)};
- if (Result.ErrorCode)
- {
- Result.Reason = fmt::format("Failed fetching oplog container build from {}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- Result.Reason);
- return Result;
- }
- CbObject BuildObject = LoadCompactBinaryObject(GetBuildResult.Response);
- if (!BuildObject)
- {
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The build {}/{}/{}/{} payload is not formatted as a compact binary object"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId);
- return Result;
- }
- CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
- if (!PartsObject)
+ LoadContainerResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
+
+ try
{
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId);
- return Result;
+ CbObject BuildObject = m_BuildStorage->GetBuild(m_BuildId);
+
+ CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
+ if (!PartsObject)
+ {
+ throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv,
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId));
+ }
+ m_OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId();
+ if (m_OplogBuildPartId == Oid::Zero)
+ {
+ throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv,
+ m_Url,
+ m_Namespace,
+ m_Bucket,
+ m_BuildId,
+ OplogContainerPartName));
+ }
+
+ Result.ContainerObject = m_BuildStorage->GetBuildPart(m_BuildId, m_OplogBuildPartId);
}
- m_OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId();
- if (m_OplogBuildPartId == Oid::Zero)
+ catch (const HttpClientError& Ex)
{
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv,
- m_JupiterClient->ServiceUrl(),
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
m_Namespace,
m_Bucket,
m_BuildId,
- OplogContainerPartName);
- return Result;
+ Ex.what());
}
-
- JupiterResult GetBuildPartResult = Session.GetBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
- AddStats(GetBuildPartResult);
- Result = {ConvertResult(GetBuildResult)};
- Result.ElapsedSeconds += GetBuildResult.ElapsedSeconds;
- if (Result.ErrorCode)
+ catch (const std::exception& Ex)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("Failed fetching oplog build part from {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
+ Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'",
+ m_Url,
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
- Result.Reason);
- return Result;
+ Ex.what());
}
- CbObject ContainerObject = LoadCompactBinaryObject(GetBuildPartResult.Response);
- if (!ContainerObject)
- {
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The build part for oplog container {}/{}/{}/{}/{} is not formatted as a compact binary object"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- m_OplogBuildPartId);
- return Result;
- }
- Result.ContainerObject = std::move(ContainerObject);
return Result;
}
virtual GetKnownBlocksResult GetKnownBlocks() override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1);
- AddStats(FindResult);
- GetKnownBlocksResult Result{ConvertResult(FindResult)};
- if (Result.ErrorCode)
+
+ GetKnownBlocksResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
+
+ try
{
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- Result.Reason);
- return Result;
+ CbObject KnownBlocks = m_BuildStorage->FindBlocks(m_BuildId, 10000u);
+ std::optional<std::vector<ChunkBlockDescription>> Blocks = ParseChunkBlockDescriptionList(KnownBlocks);
+ Result.Blocks.reserve(Blocks.value().size());
+ for (ChunkBlockDescription& BlockDescription : Blocks.value())
+ {
+ Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash,
+ .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)});
+ }
}
- if (ValidateCompactBinary(FindResult.Response.GetView(), CbValidateMode::Default) != CbValidateError::None)
+ catch (const HttpClientError& Ex)
{
- Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a compact binary object"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId);
- return Result;
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason =
+ fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
}
- std::optional<std::vector<ChunkBlockDescription>> Blocks =
- ParseChunkBlockDescriptionList(LoadCompactBinaryObject(FindResult.Response));
- if (!Blocks)
+ catch (const std::exception& Ex)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a list of blocks"sv,
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId);
- return Result;
- }
- Result.Blocks.reserve(Blocks.value().size());
- for (ChunkBlockDescription& BlockDescription : Blocks.value())
- {
- Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash,
- .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)});
+ Result.Reason =
+ fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
}
+
return Result;
}
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
- JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath);
- AddStats(GetResult);
- LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
- if (GetResult.ErrorCode)
+ LoadAttachmentResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
+
+ try
+ {
+ Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash);
+ }
+ catch (const HttpClientError& Ex)
+ {
+ Result.ErrorCode = MakeErrorCode(Ex);
+ Result.Reason =
+ fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+ }
+ catch (const std::exception& Ex)
{
- Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}/{}. Reason: '{}'",
- m_JupiterClient->ServiceUrl(),
- m_Namespace,
- m_Bucket,
- m_BuildId,
- RawHash,
- Result.Reason);
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason =
+ fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
}
+
return Result;
}
virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override
{
LoadAttachmentsResult Result;
+ Stopwatch Timer;
+ auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; });
for (const IoHash& Hash : RawHashes)
{
LoadAttachmentResult ChunkResult = LoadAttachment(Hash);
@@ -386,81 +445,27 @@ public:
}
private:
- void AddStats(const JupiterResult& Result)
- {
- m_SentBytes.fetch_add(gsl::narrow<uint64_t>(Result.SentBytes));
- m_ReceivedBytes.fetch_add(gsl::narrow<uint64_t>(Result.ReceivedBytes));
- m_RequestTimeNS.fetch_add(static_cast<uint64_t>(Result.ElapsedSeconds * 1000000000));
- SetAtomicMax(m_PeakSentBytes, Result.SentBytes);
- SetAtomicMax(m_PeakReceivedBytes, Result.ReceivedBytes);
- if (Result.ElapsedSeconds > 0.0)
- {
- uint64_t BytesPerSec = static_cast<uint64_t>((Result.SentBytes + Result.ReceivedBytes) / Result.ElapsedSeconds);
- SetAtomicMax(m_PeakBytesPerSec, BytesPerSec);
- }
-
- m_RequestCount.fetch_add(1);
- }
-
- static Result ConvertResult(const JupiterResult& Response)
+ static int MakeErrorCode(const HttpClientError& Ex)
{
- std::string Text;
- int32_t ErrorCode = 0;
- if (Response.ErrorCode != 0 || !Response.Success)
- {
- if (Response.Response)
- {
- HttpContentType ContentType = Response.Response.GetContentType();
- if (ContentType == ZenContentType::kText || ContentType == ZenContentType::kJSON)
- {
- ExtendableStringBuilder<256> SB;
- SB.Append("\n");
- SB.Append(std::string_view(reinterpret_cast<const std::string::value_type*>(Response.Response.GetData()),
- Response.Response.GetSize()));
- Text = SB.ToString();
- }
- else if (ContentType == ZenContentType::kCbObject)
- {
- ExtendableStringBuilder<256> SB;
- SB.Append("\n");
- CompactBinaryToJson(Response.Response.GetView(), SB);
- Text = SB.ToString();
- }
- }
- }
- if (Response.ErrorCode != 0)
- {
- ErrorCode = Response.ErrorCode;
- }
- else if (!Response.Success)
- {
- ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- }
- return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text};
+ return Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0;
}
- Ref<JupiterClient> m_JupiterClient;
- const std::string m_Namespace;
- const std::string m_Bucket;
- const Oid m_BuildId;
- IoBuffer m_MetaData;
- Oid m_OplogBuildPartId = Oid::Zero;
- std::filesystem::path m_TempFilePath;
- const bool m_EnableBlocks = true;
- const bool m_UseTempBlocks = true;
- const bool m_AllowRedirect = false;
-
- std::atomic_uint64_t m_SentBytes = {};
- std::atomic_uint64_t m_ReceivedBytes = {};
- std::atomic_uint64_t m_RequestTimeNS = {};
- std::atomic_uint64_t m_RequestCount = {};
- std::atomic_uint64_t m_PeakSentBytes = {};
- std::atomic_uint64_t m_PeakReceivedBytes = {};
- std::atomic_uint64_t m_PeakBytesPerSec = {};
+ std::unique_ptr<BuildStorage::Statistics> m_BuildStorageStats;
+ std::unique_ptr<HttpClient> m_BuildStorageHttp;
+ std::unique_ptr<BuildStorage> m_BuildStorage;
+ const std::string m_Url;
+ const std::string m_Namespace;
+ const std::string m_Bucket;
+ const Oid m_BuildId;
+ IoBuffer m_MetaData;
+ Oid m_OplogBuildPartId = Oid::Zero;
+ const bool m_EnableBlocks = true;
+ const bool m_UseTempBlocks = true;
+ const bool m_AllowRedirect = false;
};
std::shared_ptr<RemoteProjectStore>
-CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
+CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet)
{
std::string Url = Options.Url;
if (Url.find("://"sv) == std::string::npos)
@@ -468,13 +473,7 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
// Assume https URL
Url = fmt::format("https://{}"sv, Url);
}
- JupiterClientOptions ClientOptions{.Name = "Remote store"sv,
- .ServiceUrl = Url,
- .ConnectTimeout = std::chrono::milliseconds(2000),
- .Timeout = std::chrono::milliseconds(1800000),
- .AssumeHttp2 = Options.AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 4};
+
// 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
// 2) Access token as parameter in request
// 3) Environment variable (different win vs linux/mac)
@@ -491,7 +490,7 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
}
else if (!Options.OidcExePath.empty())
{
- if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe)
{
TokenProvider = TokenProviderMaybe.value();
}
@@ -502,16 +501,31 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
}
- Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider)));
+ HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
+ .ConnectTimeout = std::chrono::milliseconds(2000),
+ .Timeout = std::chrono::milliseconds(1800000),
+ .AccessTokenProvider = std::move(TokenProvider),
+ .AssumeHttp2 = Options.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 4};
+
+ std::unique_ptr<BuildStorage::Statistics> BuildStorageStats(std::make_unique<BuildStorage::Statistics>());
+
+ std::unique_ptr<HttpClient> BuildStorageHttp = std::make_unique<HttpClient>(Url, ClientSettings);
+
+ std::unique_ptr<BuildStorage> BuildStorage =
+ CreateJupiterBuildStorage(Log(), *BuildStorageHttp, *BuildStorageStats, Options.Namespace, Options.Bucket, false, TempFilePath);
- std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(Client),
+ std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(BuildStorageStats),
+ std::move(BuildStorageHttp),
+ std::move(BuildStorage),
+ Url,
Options.Namespace,
Options.Bucket,
Options.BuildId,
Options.MetaData,
Options.ForceDisableBlocks,
- Options.ForceDisableTempBlocks,
- TempFilePath);
+ Options.ForceDisableTempBlocks);
return RemoteStore;
}
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.h b/src/zenserver/projectstore/buildsremoteprojectstore.h
index c52b13886..60b6caef7 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.h
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.h
@@ -24,7 +24,8 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions
IoBuffer MetaData;
};
-std::shared_ptr<RemoteProjectStore> CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
- const std::filesystem::path& TempFilePath);
+std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
+ const std::filesystem::path& TempFilePath,
+ bool Quiet);
} // namespace zen
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 317a419eb..9600133f3 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -235,13 +235,11 @@ namespace {
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store,
- ProjectStore* Projects,
+HttpProjectService::HttpProjectService(ProjectStore* Projects,
HttpStatusService& StatusService,
HttpStatsService& StatsService,
AuthMgr& AuthMgr)
: m_Log(logging::Get("project"))
-, m_CidStore(Store)
, m_ProjectStore(Projects)
, m_StatusService(StatusService)
, m_StatsService(StatsService)
@@ -407,8 +405,45 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
{
ZEN_TRACE_CPU("ProjectService::Stats");
- const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
- const CidStoreSize CidSize = m_CidStore.TotalSize();
+ bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true";
+
+ const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
+ uint64_t TotalChunkHitCount = 0;
+ uint64_t TotalChunkMissCount = 0;
+ uint64_t TotalChunkWriteCount = 0;
+ CidStoreSize TotalCidSize;
+
+ tsl::robin_map<CidStore*, std::string> UniqueStores;
+ {
+ m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) {
+ CidStore* Store = &Project.GetCidStore();
+ if (auto It = UniqueStores.find(Store); It == UniqueStores.end())
+ {
+ UniqueStores.insert_or_assign(Store, Project.Identifier);
+ }
+ else
+ {
+ UniqueStores.insert_or_assign(Store, std::string{});
+ }
+ });
+
+ for (auto It : UniqueStores)
+ {
+ CidStore* ChunkStore = It.first;
+
+ CidStoreStats ChunkStoreStats = ChunkStore->Stats();
+ CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
+
+ TotalChunkHitCount += ChunkStoreStats.HitCount;
+ TotalChunkMissCount += ChunkStoreStats.MissCount;
+ TotalChunkWriteCount += ChunkStoreStats.WriteCount;
+
+ TotalCidSize.TinySize += ChunkStoreSize.TinySize;
+ TotalCidSize.SmallSize += ChunkStoreSize.SmallSize;
+ TotalCidSize.LargeSize += ChunkStoreSize.LargeSize;
+ TotalCidSize.TotalSize += ChunkStoreSize.TotalSize;
+ }
+ }
CbObjectWriter Cbo;
@@ -460,12 +495,66 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
{
Cbo.BeginObject("size");
{
- Cbo << "tiny" << CidSize.TinySize;
- Cbo << "small" << CidSize.SmallSize;
- Cbo << "large" << CidSize.LargeSize;
- Cbo << "total" << CidSize.TotalSize;
+ Cbo << "tiny" << TotalCidSize.TinySize;
+ Cbo << "small" << TotalCidSize.SmallSize;
+ Cbo << "large" << TotalCidSize.LargeSize;
+ Cbo << "total" << TotalCidSize.TotalSize;
}
Cbo.EndObject();
+
+ if (ShowCidStoreStats)
+ {
+ Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount;
+ const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount;
+ Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0);
+
+ Cbo.BeginObject("store");
+
+ auto OutputStats = [&](CidStore& ChunkStore) {
+ CidStoreStats StoreStats = ChunkStore.Stats();
+ Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount;
+ const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount;
+ Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0);
+ EmitSnapshot("read", StoreStats.FindChunkOps, Cbo);
+ EmitSnapshot("write", StoreStats.AddChunkOps, Cbo);
+ };
+
+ if (UniqueStores.size() > 1)
+ {
+ Cbo.BeginArray("projects");
+ for (auto It : UniqueStores)
+ {
+ CidStore* ChunkStore = It.first;
+ const std::string& ProjectId = It.second;
+ CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
+
+ Cbo.BeginObject();
+ {
+ Cbo << "project" << ProjectId;
+ Cbo.BeginObject("stats");
+ OutputStats(*ChunkStore);
+ Cbo.EndObject();
+
+ Cbo.BeginObject("size");
+ {
+ Cbo << "tiny" << ChunkStoreSize.TinySize;
+ Cbo << "small" << ChunkStoreSize.SmallSize;
+ Cbo << "large" << ChunkStoreSize.LargeSize;
+ Cbo << "total" << ChunkStoreSize.TotalSize;
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndArray(); // projects
+ }
+ else if (UniqueStores.size() != 0)
+ {
+ CidStore& ChunkStore = *UniqueStores.begin()->first;
+ OutputStats(ChunkStore);
+ }
+ Cbo.EndObject();
+ }
}
Cbo.EndObject();
@@ -1125,6 +1214,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
ProjectStore::Oplog& Oplog = *FoundLog;
IoBuffer Payload = HttpReq.ReadPayload();
@@ -1137,7 +1228,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
std::vector<IoHash> MissingChunks;
CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
- if (m_CidStore.ContainsChunk(Hash))
+ if (ChunkStore.ContainsChunk(Hash))
{
// Return null attachment as we already have it, no point in reading it and storing it again
return {};
@@ -1393,6 +1484,8 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
ProjectStore::Oplog& Oplog = *FoundLog;
if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
@@ -1407,7 +1500,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
Op.IterateAttachments([&](CbFieldView FieldView) {
const IoHash AttachmentHash = FieldView.AsAttachment();
- IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(AttachmentHash);
if (Payload)
{
switch (Payload.GetContentType())
@@ -2036,11 +2129,14 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
CSVHeader(Details, AttachmentDetails, CSVWriter);
m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+ CidStore& ChunkStore = Project.GetCidStore();
+
Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
+ Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN,
+ const Oid& Key,
+ CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
});
});
@@ -2054,8 +2150,9 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
m_ProjectStore->DiscoverProjects();
m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- std::vector<std::string> OpLogs = Project.ScanForOplogs();
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ CidStore& ChunkStore = Project.GetCidStore();
+ std::vector<std::string> OpLogs = Project.ScanForOplogs();
+ CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
});
}
Cbo.EndArray();
@@ -2084,7 +2181,8 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
{
return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
- ProjectStore::Project& Project = *FoundProject.Get();
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
if (CSV)
{
@@ -2092,10 +2190,11 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
CSVHeader(Details, AttachmentDetails, CSVWriter);
FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
+ Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN,
+ const Oid& Key,
+ CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
});
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
@@ -2105,7 +2204,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
Cbo.BeginArray("projects");
{
- CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2141,16 +2240,17 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
+ ProjectStore::Oplog& Oplog = *FoundLog;
if (CSV)
{
ExtendableStringBuilder<4096> CSVWriter;
CSVHeader(Details, AttachmentDetails, CSVWriter);
Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ [this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
});
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
@@ -2159,7 +2259,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
CbObjectWriter Cbo;
Cbo.BeginArray("oplogs");
{
- CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+ CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2204,9 +2304,10 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId));
}
- const Oid ObjId = Oid::FromHexString(OpId);
- ProjectStore::Project& Project = *FoundProject.Get();
- ProjectStore::Oplog& Oplog = *FoundLog;
+ const Oid ObjId = Oid::FromHexString(OpId);
+ ProjectStore::Project& Project = *FoundProject.Get();
+ CidStore& ChunkStore = Project.GetCidStore();
+ ProjectStore::Oplog& Oplog = *FoundLog;
std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId);
if (!Op.has_value())
@@ -2224,7 +2325,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
ExtendableStringBuilder<4096> CSVWriter;
CSVHeader(Details, AttachmentDetails, CSVWriter);
- CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
+ CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
else
@@ -2232,7 +2333,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
CbObjectWriter Cbo;
Cbo.BeginArray("ops");
{
- CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
+ CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 295defa5c..5782188e6 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -35,11 +35,7 @@ class ProjectStore;
class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpProjectService(CidStore& Store,
- ProjectStore* InProjectStore,
- HttpStatusService& StatusService,
- HttpStatsService& StatsService,
- AuthMgr& AuthMgr);
+ HttpProjectService(ProjectStore* InProjectStore, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr);
~HttpProjectService();
virtual const char* BaseUri() const override;
@@ -92,7 +88,6 @@ private:
inline LoggerRef Log() { return m_Log; }
LoggerRef m_Log;
- CidStore& m_CidStore;
HttpRequestRouter m_Router;
Ref<ProjectStore> m_ProjectStore;
HttpStatusService& m_StatusService;
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 3728babb5..dba5cd4a7 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -337,7 +337,7 @@ private:
};
std::shared_ptr<RemoteProjectStore>
-CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
+CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet)
{
std::string Url = Options.Url;
if (Url.find("://"sv) == std::string::npos)
@@ -368,7 +368,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
}
else if (!Options.OidcExePath.empty())
{
- if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe)
{
TokenProvider = TokenProviderMaybe.value();
}
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h
index 8bf79d563..ac2d25b47 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.h
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h
@@ -25,6 +25,7 @@ struct JupiterRemoteStoreOptions : RemoteStoreOptions
};
std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
- const std::filesystem::path& TempFilePath);
+ const std::filesystem::path& TempFilePath,
+ bool Quiet);
} // namespace zen
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 53e687983..d6dd6ef9b 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -267,7 +267,7 @@ namespace {
ForceDisableBlocks,
ForceDisableTempBlocks,
AssumeHttp2};
- RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath);
+ RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false);
}
if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -364,7 +364,7 @@ namespace {
ForceDisableTempBlocks,
AssumeHttp2,
MetaData};
- RemoteStore = CreateBuildsRemoteStore(Options, TempFilePath);
+ RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false);
}
if (!RemoteStore)
@@ -1176,7 +1176,7 @@ ProjectStore::Oplog::Flush()
}
void
-ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx)
+ProjectStore::Oplog::Scrub(ScrubContext& Ctx)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
@@ -1492,7 +1492,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f
if (ValidationError != CbValidateError::None)
{
- ZEN_ERROR("validation error {} hit for oplog config at '{}'", int(ValidationError), StateFilePath);
+ ZEN_ERROR("validation error {} hit for oplog config at '{}'", ToString(ValidationError), StateFilePath);
return CbObject();
}
@@ -3168,7 +3168,7 @@ ProjectStore::Project::Read()
}
else
{
- ZEN_ERROR("validation error {} hit for '{}'", int(ValidationError), ProjectStateFilePath);
+ ZEN_ERROR("validation error {} hit for '{}'", ToString(ValidationError), ProjectStateFilePath);
}
ReadAccessTimes();
@@ -3260,7 +3260,7 @@ ProjectStore::Project::ReadAccessTimes()
}
else
{
- ZEN_WARN("project '{}': validation error {} hit for '{}'", Identifier, int(ValidationError), ProjectAccessTimesFilePath);
+ ZEN_WARN("project '{}': validation error {} hit for '{}'", Identifier, ToString(ValidationError), ProjectAccessTimesFilePath);
}
}
@@ -3575,7 +3575,7 @@ ProjectStore::Project::Flush()
}
void
-ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
+ProjectStore::Project::Scrub(ScrubContext& Ctx)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
// Scrubbing needs to check all existing oplogs
@@ -3587,7 +3587,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) {
if (!IsExpired(GcClock::TimePoint::min(), Ops))
{
- Ops.ScrubStorage(Ctx);
+ Ops.Scrub(Ctx);
}
});
}
@@ -3832,7 +3832,7 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const
//////////////////////////////////////////////////////////////////////////
-ProjectStore::ProjectStore(CidStore& Store,
+ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore,
std::filesystem::path BasePath,
GcManager& Gc,
JobQueue& JobQueue,
@@ -3840,7 +3840,7 @@ ProjectStore::ProjectStore(CidStore& Store,
const Configuration& Config)
: m_Log(logging::Get("project"))
, m_Gc(Gc)
-, m_CidStore(Store)
+, m_GetCidStore(std::move(GetCidStore))
, m_JobQueue(JobQueue)
, m_OpenProcessCache(InOpenProcessCache)
, m_ProjectBasePath(BasePath)
@@ -3973,7 +3973,7 @@ ProjectStore::ScrubStorage(ScrubContext& Ctx)
}
for (const Ref<Project>& Project : Projects)
{
- Project->ScrubStorage(Ctx);
+ Project->Scrub(Ctx);
}
}
@@ -4025,6 +4025,8 @@ ProjectStore::OpenProject(std::string_view ProjectId)
}
}
+ CidStore& ChunkStore = m_GetCidStore(ProjectId);
+
RwLock::ExclusiveLockScope _(m_ProjectsLock);
if (auto ProjIt = m_Projects.find(std::string{ProjectId}); ProjIt != m_Projects.end())
{
@@ -4041,7 +4043,7 @@ ProjectStore::OpenProject(std::string_view ProjectId)
Ref<Project>& Prj =
m_Projects
- .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+ .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath)))
.first->second;
Prj->Identifier = ProjectId;
Prj->Read();
@@ -4068,12 +4070,14 @@ ProjectStore::NewProject(const std::filesystem::path& BasePath,
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Store::NewProject");
+ CidStore& ChunkStore = m_GetCidStore(ProjectId);
+
RwLock::ExclusiveLockScope _(m_ProjectsLock);
ZEN_INFO("project '{}': creating project at '{}'", ProjectId, BasePath);
Ref<Project>& Prj =
- m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+ m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath)))
.first->second;
Prj->Identifier = ProjectId;
Prj->RootDir = RootDir;
@@ -4802,7 +4806,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
}
const IoHash Hash = IoHash::FromHexString(Cid);
- OutChunk = m_CidStore.FindChunkByCid(Hash);
+ OutChunk = Project->GetCidStore().FindChunkByCid(Hash);
if (!OutChunk)
{
@@ -4865,7 +4869,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
}
FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash});
- CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash);
+ CidStore::InsertResult Result = Project->GetCidStore().AddChunk(Chunk, Hash);
return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}};
}
@@ -4894,18 +4898,19 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
if (RequestObject["chunks"sv].IsArray())
{
// Legacy full chunks only by rawhash
- CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
-
+ CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
CbObjectWriter ResponseWriter;
ResponseWriter.BeginArray("chunks"sv);
for (CbFieldView FieldView : ChunksArray)
{
IoHash RawHash = FieldView.AsHash();
- IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash);
+ IoBuffer ChunkBuffer = ChunkStore.FindChunkByCid(RawHash);
if (ChunkBuffer)
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer));
@@ -5057,7 +5062,7 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
if (ChunkRequest.Input.Id.index() == 0)
{
const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id);
- IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash);
+ IoBuffer Payload = ChunkStore.FindChunkByCid(ChunkHash);
if (Payload)
{
ChunkRequest.Output.Exists = true;
@@ -5244,7 +5249,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
return {HttpResponseCode::BadRequest, "Invalid payload format"};
}
- CidStore& ChunkStore = m_CidStore;
+ CidStore& ChunkStore = Project->GetCidStore();
RwLock AttachmentsLock;
tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
@@ -5350,7 +5355,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId,
}
}
- CidStore& ChunkStore = m_CidStore;
+ CidStore& ChunkStore = Project->GetCidStore();
RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer(
ChunkStore,
@@ -5462,6 +5467,8 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Project->TouchOplog(OplogId);
+ CidStore& ChunkStore = Project->GetCidStore();
+
if (Method == "import"sv)
{
if (!AreDiskWritesAllowed())
@@ -5543,7 +5550,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Oplog->CaptureAddedAttachments(WriteRawHashes);
- m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
+ ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
}
HttpReq.WriteResponse(HttpResponseCode::OK);
return true;
@@ -5716,14 +5723,14 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
ResponseObj.EndArray();
}
- // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new
- // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have
+ // Ops that have moved chunks to a compressed buffer for storage in ChunkStore have been rewritten with references to the new
+ // chunk(s). Make sure we add the chunks to ChunkStore, and do it after we update the oplog so GC doesn't think we have
// unreferenced chunks.
for (auto It : AddedChunks)
{
const IoHash& RawHash = It.first;
AddedChunk& Chunk = It.second;
- CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash);
+ CidStore::InsertResult Result = ChunkStore.AddChunk(Chunk.Buffer, RawHash);
if (Result.New)
{
InlinedBytes += Chunk.RawSize;
@@ -5786,7 +5793,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
EmbedLooseFile,
Force,
IgnoreMissingAttachments](JobContext& Context) {
- RemoteProjectStore::Result Result = SaveOplog(m_CidStore,
+ RemoteProjectStore::Result Result = SaveOplog(Project->GetCidStore(),
*ActualRemoteStore,
*Project.Get(),
*OplogPtr,
@@ -5801,7 +5808,8 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
{
- throw std::runtime_error(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second);
+ throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second,
+ (int)Response.first);
}
});
@@ -5830,19 +5838,26 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
}
std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store);
RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
+ CidStore& ChunkStore = Project.GetCidStore();
ZEN_INFO("Loading oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description);
JobId JobId = m_JobQueue.QueueJob(
fmt::format("Import oplog '{}/{}'", Project.Identifier, Oplog.OplogId()),
- [this, ActualRemoteStore = std::move(RemoteStore), OplogPtr = &Oplog, Force, IgnoreMissingAttachments, CleanOplog](
- JobContext& Context) {
+ [this,
+ ChunkStore = &ChunkStore,
+ ActualRemoteStore = std::move(RemoteStore),
+ OplogPtr = &Oplog,
+ Force,
+ IgnoreMissingAttachments,
+ CleanOplog](JobContext& Context) {
RemoteProjectStore::Result Result =
- LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context);
+ LoadOplog(*ChunkStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context);
auto Response = ConvertResult(Result);
ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
{
- throw std::runtime_error(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second);
+ throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second,
+ (int)Response.first);
}
});
@@ -6875,6 +6890,11 @@ namespace testutils {
return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags);
}
+ ProjectStore::GetCidStoreFunc SingleChunkStore(CidStore& ChunkStore)
+ {
+ return [ChunkStore = &ChunkStore](std::string_view) -> CidStore& { return *ChunkStore; };
+ }
+
} // namespace testutils
TEST_CASE("project.opkeys")
@@ -6937,13 +6957,18 @@ TEST_CASE("project.store.create")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::string_view ProjectName("proj1"sv);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -6968,12 +6993,17 @@ TEST_CASE("project.store.lifetimes")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -7031,12 +7061,17 @@ TEST_CASE_TEMPLATE("project.store.export",
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -7070,7 +7105,7 @@ TEST_CASE_TEMPLATE("project.store.export",
std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options);
RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
- RemoteProjectStore::Result ExportResult = SaveOplog(CidStore,
+ RemoteProjectStore::Result ExportResult = SaveOplog(ChunkStore,
*RemoteStore,
*Project.Get(),
*Oplog,
@@ -7087,7 +7122,7 @@ TEST_CASE_TEMPLATE("project.store.export",
ProjectStore::Oplog* OplogImport = Project->NewOplog("oplog2", {});
CHECK(OplogImport != nullptr);
- RemoteProjectStore::Result ImportResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ false,
@@ -7096,7 +7131,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportForceResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportForceResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ true,
@@ -7105,7 +7140,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportForceResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportCleanResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportCleanResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ false,
@@ -7114,7 +7149,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportCleanResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(CidStore,
+ RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(ChunkStore,
*RemoteStore,
*OplogImport,
/*Force*/ true,
@@ -7134,12 +7169,17 @@ TEST_CASE("project.store.gc")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
@@ -7335,12 +7375,17 @@ TEST_CASE("project.store.gc.prep")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
@@ -7383,7 +7428,7 @@ TEST_CASE("project.store.gc.prep")
// Equivalent of a `prep` existance check call
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7397,7 +7442,7 @@ TEST_CASE("project.store.gc.prep")
// If a gc comes in between our prep and op write the chunks will be removed
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7417,7 +7462,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7431,7 +7476,7 @@ TEST_CASE("project.store.gc.prep")
// Attachments should now be retained
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7445,7 +7490,7 @@ TEST_CASE("project.store.gc.prep")
// Attachments should now be retained across multiple GCs if retain time is still valud
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7457,7 +7502,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
@@ -7474,7 +7519,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7507,7 +7552,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
Sleep(200);
@@ -7522,7 +7567,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
// This pass the retention time has expired and the last GC pass cleared the entries
@@ -7536,7 +7581,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
}
@@ -7550,12 +7595,17 @@ TEST_CASE("project.store.rpc.getchunks")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
@@ -8472,12 +8522,17 @@ TEST_CASE("project.store.partial.read")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
@@ -8650,12 +8705,17 @@ TEST_CASE("project.store.iterateoplog")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore CidStore(Gc);
+ CidStore ChunkStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- CidStore.Initialize(CidConfig);
+ ChunkStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
+ ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
+ BasePath,
+ Gc,
+ *JobQueue,
+ ProcessCache,
+ ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv";
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 368da5ea4..eb27665f9 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -68,7 +68,9 @@ public:
{
};
- ProjectStore(CidStore& Store,
+ typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc;
+
+ ProjectStore(GetCidStoreFunc&& GetCidStore,
std::filesystem::path BasePath,
GcManager& Gc,
JobQueue& JobQueue,
@@ -156,7 +158,7 @@ public:
LoggerRef Log() { return m_OuterProject->Log(); }
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
+ void Scrub(ScrubContext& Ctx);
static uint64_t TotalSize(const std::filesystem::path& BasePath);
uint64_t TotalSize() const;
@@ -326,11 +328,13 @@ public:
Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath);
virtual ~Project();
+ CidStore& GetCidStore() { return m_CidStore; };
+
void Read();
void Write();
[[nodiscard]] static bool Exists(const std::filesystem::path& BasePath);
void Flush();
- void ScrubStorage(ScrubContext& Ctx);
+ void Scrub(ScrubContext& Ctx);
LoggerRef Log() const;
static uint64_t TotalSize(const std::filesystem::path& BasePath);
uint64_t TotalSize() const;
@@ -405,6 +409,7 @@ public:
LoggerRef Log() { return m_Log; }
const std::filesystem::path& BasePath() const { return m_ProjectBasePath; }
+ // GcStorage
virtual void ScrubStorage(ScrubContext& Ctx) override;
virtual GcStorageSize StorageSize() const override;
@@ -498,7 +503,7 @@ public:
private:
LoggerRef m_Log;
GcManager& m_Gc;
- CidStore& m_CidStore;
+ GetCidStoreFunc m_GetCidStore;
JobQueue& m_JobQueue;
OpenProcessCache& m_OpenProcessCache;
std::filesystem::path m_ProjectBasePath;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-22 13:37:49 +0000