aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver')
-rw-r--r--src/zenserver/main.cpp69
-rw-r--r--src/zenserver/objectstore/objectstore.cpp29
-rw-r--r--src/zenserver/objectstore/objectstore.h4
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.cpp38
-rw-r--r--src/zenserver/projectstore/fileremoteprojectstore.cpp4
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp32
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.cpp2
-rw-r--r--src/zenserver/projectstore/projectstore.cpp51
-rw-r--r--src/zenserver/projectstore/remoteprojectstore.cpp127
-rw-r--r--src/zenserver/projectstore/remoteprojectstore.h6
-rw-r--r--src/zenserver/windows/service.cpp8
-rw-r--r--src/zenserver/windows/service.h4
-rw-r--r--src/zenserver/workspaces/httpworkspaces.cpp12
13 files changed, 217 insertions, 169 deletions
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index d5419d342..9ae54bdf1 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -23,6 +23,8 @@
#include <zencore/memory/memorytrace.h>
#include <zencore/memory/newdelete.h>
+#include <zenutil/service.h>
+
#include "config.h"
#include "diag/logging.h"
#include "sentryintegration.h"
@@ -32,6 +34,10 @@
# include "windows/service.h"
#endif
+#if ZEN_PLATFORM_LINUX
+# include <systemd/sd-daemon.h>
+#endif
+
//////////////////////////////////////////////////////////////////////////
// We don't have any doctest code in this file but this is needed to bring
// in some shared code into the executable
@@ -88,9 +94,47 @@ ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) : m_ServerOptions(
{
}
+void
+ReportServiceStatus(ServiceStatus Status)
+{
+#if ZEN_PLATFORM_WINDOWS
+ switch (Status)
+ {
+ case ServiceStatus::Starting:
+ ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, 3000);
+ break;
+ case ServiceStatus::Running:
+ ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0);
+ break;
+ case ServiceStatus::Stopping:
+ ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0);
+ break;
+ case ServiceStatus::Stopped:
+ ReportSvcStatus(SERVICE_STOPPED, (DWORD)ApplicationExitCode(), 0);
+ break;
+ default:
+ break;
+ }
+#elif ZEN_PLATFORM_LINUX
+ switch (Status)
+ {
+ case ServiceStatus::Running:
+ sd_notify(0, "READY=1");
+ break;
+ case ServiceStatus::Stopping:
+ sd_notify(0, "STOPPING=1");
+ break;
+ case ServiceStatus::Stopped:
+ sd_notifyf(0, "EXIT_STATUS=%d", ApplicationExitCode());
+ break;
+ }
+#endif
+}
+
int
ZenEntryPoint::Run()
{
+ ZEN_INFO("ZenEntryPoint::Run()");
zen::SetCurrentThreadName("main");
#if ZEN_USE_SENTRY
@@ -107,8 +151,11 @@ ZenEntryPoint::Run()
try
{
// Mutual exclusion and synchronization
+ ZEN_INFO("ZenServerState ServerState");
ZenServerState ServerState;
+ ZEN_INFO("ServerState.Initialize()");
ServerState.Initialize();
+ ZEN_INFO("ServerState.Sweep()");
ServerState.Sweep();
uint32_t AttachSponsorProcessRetriesLeft = 3;
@@ -173,6 +220,8 @@ ZenEntryPoint::Run()
}
}
+ ZEN_INFO("Preparing lock file");
+
std::error_code Ec;
std::filesystem::path LockFilePath = m_ServerOptions.DataDir / ".lock";
@@ -186,6 +235,7 @@ ZenEntryPoint::Run()
.ExecutablePath = GetRunningExecutablePath()});
};
+ ZEN_INFO("m_LockFile.Create");
m_LockFile.Create(LockFilePath, MakeLockData(false), Ec);
if (Ec)
@@ -201,6 +251,7 @@ ZenEntryPoint::Run()
}
}
+ ZEN_INFO("InitializeServerLogging");
InitializeServerLogging(m_ServerOptions);
ZEN_INFO("Command line: {}", m_ServerOptions.CommandLine);
@@ -273,6 +324,8 @@ ZenEntryPoint::Run()
}});
auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] {
+ ReportServiceStatus(ServiceStatus::Stopping);
+
if (ShutdownEvent)
{
ShutdownEvent->Set();
@@ -294,6 +347,8 @@ ZenEntryPoint::Run()
NamedEvent ParentEvent{m_ServerOptions.ChildId};
ParentEvent.Set();
}
+
+ ReportServiceStatus(ServiceStatus::Running);
});
Server.Run();
@@ -303,6 +358,14 @@ ZenEntryPoint::Run()
ZEN_CRITICAL("Caught assert exception in main for process {}: {}", zen::GetCurrentProcessId(), AssertEx.FullDescription());
RequestApplicationExit(1);
}
+ catch (const std::system_error& e)
+ {
+ ZEN_CRITICAL("Caught system error exception in main for process {}: {} ({})",
+ zen::GetCurrentProcessId(),
+ e.what(),
+ e.code().value());
+ RequestApplicationExit(1);
+ }
catch (const std::exception& e)
{
ZEN_CRITICAL("Caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what());
@@ -311,6 +374,8 @@ ZenEntryPoint::Run()
ShutdownServerLogging();
+ ReportServiceStatus(ServiceStatus::Stopped);
+
return ApplicationExitCode();
}
@@ -380,6 +445,10 @@ main(int argc, char* argv[])
signal(SIGINT, utils::SignalCallbackHandler);
signal(SIGTERM, utils::SignalCallbackHandler);
+#if ZEN_PLATFORM_LINUX
+ IgnoreChildSignals();
+#endif
+
try
{
ZenServerOptions ServerOptions;
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index b0212ab07..e757ef84e 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -269,9 +269,9 @@ HttpObjectStoreService::Inititalize()
m_Router.RegisterRoute(
"bucket/{path}",
[this](zen::HttpRouterRequest& Request) {
- const std::string Path = Request.GetCapture(1);
- const auto Sep = Path.find_last_of('.');
- const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0;
+ const std::string_view Path = Request.GetCapture(1);
+ const auto Sep = Path.find_last_of('.');
+ const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0;
if (IsObject)
{
@@ -337,18 +337,18 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
}
void
-HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string& Path)
+HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path)
{
namespace fs = std::filesystem;
- const auto Sep = Path.find_first_of('/');
- const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep);
+ const auto Sep = Path.find_first_of('/');
+ const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)};
if (BucketName.empty())
{
return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
}
- std::string BucketPrefix = Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1);
+ std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)};
if (BucketPrefix.empty())
{
const auto QueryParms = Request.ServerRequest().GetQueryParams();
@@ -376,7 +376,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
Writer.BeginArray("Contents"sv);
}
- void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
+ void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override
{
const fs::path FullPath = Parent / fs::path(File);
fs::path RelativePath = fs::relative(FullPath, BucketPath);
@@ -450,14 +450,13 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
}
void
-HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string& Path)
+HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path)
{
namespace fs = std::filesystem;
- const auto Sep = Path.find_first_of('/');
- const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep);
- const std::string BucketPrefix =
- Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1);
+ const auto Sep = Path.find_first_of('/');
+ const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)};
+ const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)};
const fs::path BucketDir = GetBucketDirectory(BucketName);
@@ -554,8 +553,8 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
{
namespace fs = std::filesystem;
- const std::string& BucketName = Request.GetCapture(1);
- const fs::path BucketDir = GetBucketDirectory(BucketName);
+ const std::string_view BucketName = Request.GetCapture(1);
+ const fs::path BucketDir = GetBucketDirectory(BucketName);
if (BucketDir.empty())
{
diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h
index c905ceab3..dae979c4c 100644
--- a/src/zenserver/objectstore/objectstore.h
+++ b/src/zenserver/objectstore/objectstore.h
@@ -36,9 +36,9 @@ private:
void Inititalize();
std::filesystem::path GetBucketDirectory(std::string_view BucketName);
void CreateBucket(zen::HttpRouterRequest& Request);
- void ListBucket(zen::HttpRouterRequest& Request, const std::string& Path);
+ void ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path);
void DeleteBucket(zen::HttpRouterRequest& Request);
- void GetObject(zen::HttpRouterRequest& Request, const std::string& Path);
+ void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path);
void PutObject(zen::HttpRouterRequest& Request);
ObjectStoreConfig m_Cfg;
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index 412769174..fbb9bc344 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -123,18 +123,17 @@ public:
JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
JupiterResult PutResult =
- Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, ZenContentType::kCompressedBinary, Payload);
+ Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
AddStats(PutResult);
SaveAttachmentResult Result{ConvertResult(PutResult)};
if (Result.ErrorCode)
{
- Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}/{}. Reason: '{}'",
+ Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}. Reason: '{}'",
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
RawHash,
Result.Reason);
return Result;
@@ -147,18 +146,16 @@ public:
IoBuffer MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()).GetBuffer().AsIoBuffer();
MetaPayload.SetContentType(ZenContentType::kCbObject);
- JupiterResult PutMetaResult =
- Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, MetaPayload);
+ JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, RawHash, MetaPayload);
AddStats(PutMetaResult);
RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult);
if (MetaDataResult.ErrorCode)
{
- ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}/{}. Reason: '{}'",
+ ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
RawHash,
MetaDataResult.Reason);
}
@@ -311,30 +308,28 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
- JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
+ JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId);
AddStats(FindResult);
GetKnownBlocksResult Result{ConvertResult(FindResult)};
if (Result.ErrorCode)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}/{}. Reason: '{}'",
+ Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
Result.Reason);
return Result;
}
if (ValidateCompactBinary(FindResult.Response.GetView(), CbValidateMode::Default) != CbValidateError::None)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a compact binary object"sv,
+ Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a compact binary object"sv,
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
- m_BuildId,
- m_OplogBuildPartId);
+ m_BuildId);
return Result;
}
std::optional<std::vector<ChunkBlockDescription>> Blocks =
@@ -342,15 +337,19 @@ public:
if (!Blocks)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a list of blocks"sv,
+ Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a list of blocks"sv,
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
- m_BuildId,
- m_OplogBuildPartId);
+ m_BuildId);
return Result;
}
- Result.Blocks = std::move(Blocks.value());
+ Result.Blocks.reserve(Blocks.value().size());
+ for (ChunkBlockDescription& BlockDescription : Blocks.value())
+ {
+ Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash,
+ .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)});
+ }
return Result;
}
@@ -358,18 +357,17 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
- JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath);
+ JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath);
AddStats(GetResult);
LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
if (GetResult.ErrorCode)
{
- Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}&{}/{}/{}. Reason: '{}'",
+ Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}/{}. Reason: '{}'",
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
RawHash,
Result.Reason);
}
diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index 5a21a7540..98e292d91 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -192,8 +192,8 @@ public:
return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent),
.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}};
}
- std::vector<ChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes);
- GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}};
+ std::vector<ThinChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes);
+ GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}};
Result.Blocks = std::move(KnownBlocks);
return Result;
}
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 0b8e5f13b..47748dd90 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -983,15 +983,19 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
IoBuffer Payload = HttpReq.ReadPayload();
CbObject RequestObject = LoadCompactBinaryObject(Payload);
- std::vector<IoHash> ChunkList;
- CbArrayView HaveList = RequestObject["have"sv].AsArrayView();
- ChunkList.reserve(HaveList.Num());
- for (auto& Entry : HaveList)
+ std::vector<IoHash> NeedList;
+
{
- ChunkList.push_back(Entry.AsHash());
- }
+ eastl::fixed_vector<IoHash, 16> ChunkList;
+ CbArrayView HaveList = RequestObject["have"sv].AsArrayView();
+ ChunkList.reserve(HaveList.Num());
+ for (auto& Entry : HaveList)
+ {
+ ChunkList.push_back(Entry.AsHash());
+ }
- std::vector<IoHash> NeedList = FoundLog->CheckPendingChunkReferences(ChunkList, std::chrono::minutes(2));
+ NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2));
+ }
CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1);
Cbo.BeginArray("need");
@@ -1151,7 +1155,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
}
- std::vector<IoHash> ReferencedChunks;
+ eastl::fixed_vector<IoHash, 16> ReferencedChunks;
Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); });
// Write core to oplog
@@ -1169,7 +1173,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
// Once we stored the op, we no longer need to retain any chunks this op references
if (!ReferencedChunks.empty())
{
- FoundLog->RemovePendingChunkReferences(ReferencedChunks);
+ FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks)));
}
m_ProjectStats.OpWriteCount++;
@@ -1301,9 +1305,9 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
HttpServerRequest& HttpReq = Req.ServerRequest();
- const std::string& ProjectId = Req.GetCapture(1);
- const std::string& OplogId = Req.GetCapture(2);
- const std::string& OpIdString = Req.GetCapture(3);
+ const std::string_view ProjectId = Req.GetCapture(1);
+ const std::string_view OplogId = Req.GetCapture(2);
+ const std::string_view OpIdString = Req.GetCapture(3);
Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
if (!Project)
@@ -1690,8 +1694,8 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req)
using namespace std::literals;
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const std::string ProjectId = Req.GetCapture(1);
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ const std::string_view ProjectId = Req.GetCapture(1);
switch (HttpReq.RequestVerb())
{
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 2b6a437d1..e5839ad3b 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -193,7 +193,7 @@ public:
return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent),
.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds}};
}
- std::vector<ChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes);
+ std::vector<ThinChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes);
GetKnownBlocksResult Result{
{.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000.0}};
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index f6f7eba99..86791e29a 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -423,9 +423,13 @@ ComputeOpKey(const CbObjectView& Op)
{
using namespace std::literals;
- BinaryWriter KeyStream;
+ eastl::fixed_vector<uint8_t, 256> KeyData;
- Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyStream.Write(Data, Size); });
+ Op["key"sv].WriteToStream([&](const void* Data, size_t Size) {
+ auto Begin = reinterpret_cast<const uint8_t*>(Data);
+ auto End = Begin + Size;
+ KeyData.insert(KeyData.end(), Begin, End);
+ });
XXH3_128 KeyHash128;
@@ -434,15 +438,15 @@ ComputeOpKey(const CbObjectView& Op)
// path but longer paths are evaluated properly. In the future all key lengths
// should be evaluated using the proper path, this is a temporary workaround to
// maintain compatibility with existing disk state.
- if (KeyStream.GetSize() < 240)
+ if (KeyData.size() < 240)
{
XXH3_128Stream_deprecated KeyHasher;
- KeyHasher.Append(KeyStream.Data(), KeyStream.Size());
+ KeyHasher.Append(KeyData.data(), KeyData.size());
KeyHash128 = KeyHasher.GetHash();
}
else
{
- KeyHash128 = XXH3_128::HashMemory(KeyStream.GetView());
+ KeyHash128 = XXH3_128::HashMemory(KeyData.data(), KeyData.size());
}
Oid KeyHash;
@@ -2735,7 +2739,7 @@ ProjectStore::Oplog::CheckPendingChunkReferences(std::span<const IoHash> ChunkHa
MissingChunks.reserve(ChunkHashes.size());
for (const IoHash& FileHash : ChunkHashes)
{
- if (IoBuffer Payload = m_CidStore.FindChunkByCid(FileHash); !Payload)
+ if (!m_CidStore.ContainsChunk(FileHash))
{
MissingChunks.push_back(FileHash);
}
@@ -3359,7 +3363,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Store::OpenOplog");
- std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
{
RwLock::SharedLockScope ProjectLock(m_ProjectLock);
@@ -3367,21 +3370,35 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo
if (OplogIt != m_Oplogs.end())
{
- if (!VerifyPathOnDisk || Oplog::ExistsAt(OplogBasePath))
+ bool ReOpen = false;
+
+ if (VerifyPathOnDisk)
{
- return OplogIt->second.get();
+ std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
+
+ if (!Oplog::ExistsAt(OplogBasePath))
+ {
+ // Somebody deleted the oplog on disk behind our back
+ ProjectLock.ReleaseNow();
+ std::filesystem::path DeletePath;
+ if (!RemoveOplog(OplogId, DeletePath))
+ {
+ ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+ }
+
+ ReOpen = true;
+ }
}
- // Somebody deleted the oplog on disk behind our back
- ProjectLock.ReleaseNow();
- std::filesystem::path DeletePath;
- if (!RemoveOplog(OplogId, DeletePath))
+ if (!ReOpen)
{
- ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+ return OplogIt->second.get();
}
}
}
+ std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
+
RwLock::ExclusiveLockScope Lock(m_ProjectLock);
if (auto It = m_Oplogs.find(std::string{OplogId}); It != m_Oplogs.end())
{
@@ -8628,7 +8645,11 @@ TEST_CASE("project.store.block")
}
ChunkBlockDescription Block;
CompressedBuffer BlockBuffer = GenerateChunkBlock(std::move(Chunks), Block);
- CHECK(IterateChunkBlock(BlockBuffer.Decompress(), [](CompressedBuffer&&, const IoHash&) {}));
+ uint64_t HeaderSize;
+ CHECK(IterateChunkBlock(
+ BlockBuffer.Decompress(),
+ [](CompressedBuffer&&, const IoHash&) {},
+ HeaderSize));
}
TEST_CASE("project.store.iterateoplog")
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 0285cc22f..a7263da83 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -516,21 +516,23 @@ namespace remotestore_impl {
return;
}
- bool StoreChunksOK = IterateChunkBlock(
- BlockPayload,
- [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info](CompressedBuffer&& Chunk,
- const IoHash& AttachmentRawHash) {
- if (WantedChunks.contains(AttachmentRawHash))
- {
- WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer());
- IoHash RawHash;
- uint64_t RawSize;
- ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), RawHash, RawSize));
- ZEN_ASSERT(RawHash == AttachmentRawHash);
- WriteRawHashes.emplace_back(AttachmentRawHash);
- WantedChunks.erase(AttachmentRawHash);
- }
- });
+ uint64_t BlockHeaderSize = 0;
+ bool StoreChunksOK = IterateChunkBlock(
+ BlockPayload,
+ [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info](CompressedBuffer&& Chunk,
+ const IoHash& AttachmentRawHash) {
+ if (WantedChunks.contains(AttachmentRawHash))
+ {
+ WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer());
+ IoHash RawHash;
+ uint64_t RawSize;
+ ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), RawHash, RawSize));
+ ZEN_ASSERT(RawHash == AttachmentRawHash);
+ WriteRawHashes.emplace_back(AttachmentRawHash);
+ WantedChunks.erase(AttachmentRawHash);
+ }
+ },
+ BlockHeaderSize);
if (!StoreChunksOK)
{
@@ -1101,11 +1103,11 @@ GetBlockHashesFromOplog(CbObjectView ContainerObject)
return BlockHashes;
}
-std::vector<ChunkBlockDescription>
+std::vector<ThinChunkBlockDescription>
GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes)
{
using namespace std::literals;
- std::vector<ChunkBlockDescription> Result;
+ std::vector<ThinChunkBlockDescription> Result;
CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView();
tsl::robin_set<IoHash, IoHash::Hasher> IncludeSet;
IncludeSet.insert(IncludeBlockHashes.begin(), IncludeBlockHashes.end());
@@ -1128,7 +1130,7 @@ GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> Include
{
ChunkHashes.push_back(ChunkField.AsHash());
}
- Result.push_back({.BlockHash = BlockHash, .ChunkHashes = std::move(ChunkHashes)});
+ Result.push_back(ThinChunkBlockDescription{.BlockHash = BlockHash, .ChunkRawHashes = std::move(ChunkHashes)});
}
}
return Result;
@@ -1144,7 +1146,7 @@ BuildContainer(CidStore& ChunkStore,
bool BuildBlocks,
bool IgnoreMissingAttachments,
bool AllowChunking,
- const std::vector<ChunkBlockDescription>& KnownBlocks,
+ const std::vector<ThinChunkBlockDescription>& KnownBlocks,
WorkerThreadPool& WorkerPool,
const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock,
const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment,
@@ -1386,7 +1388,7 @@ BuildContainer(CidStore& ChunkStore,
return {};
}
- auto FindReuseBlocks = [](const std::vector<ChunkBlockDescription>& KnownBlocks,
+ auto FindReuseBlocks = [](const std::vector<ThinChunkBlockDescription>& KnownBlocks,
const std::unordered_set<IoHash, IoHash::Hasher>& Attachments,
JobContext* OptionalContext) -> std::vector<size_t> {
std::vector<size_t> ReuseBlockIndexes;
@@ -1399,14 +1401,14 @@ BuildContainer(CidStore& ChunkStore,
for (size_t KnownBlockIndex = 0; KnownBlockIndex < KnownBlocks.size(); KnownBlockIndex++)
{
- const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
- size_t BlockAttachmentCount = KnownBlock.ChunkHashes.size();
+ const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
+ size_t BlockAttachmentCount = KnownBlock.ChunkRawHashes.size();
if (BlockAttachmentCount == 0)
{
continue;
}
size_t FoundAttachmentCount = 0;
- for (const IoHash& KnownHash : KnownBlock.ChunkHashes)
+ for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes)
{
if (Attachments.contains(KnownHash))
{
@@ -1447,8 +1449,8 @@ BuildContainer(CidStore& ChunkStore,
std::vector<size_t> ReusedBlockIndexes = FindReuseBlocks(KnownBlocks, FoundHashes, OptionalContext);
for (size_t KnownBlockIndex : ReusedBlockIndexes)
{
- const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
- for (const IoHash& KnownHash : KnownBlock.ChunkHashes)
+ const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
+ for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes)
{
if (UploadAttachments.erase(KnownHash) == 1)
{
@@ -1466,10 +1468,7 @@ BuildContainer(CidStore& ChunkStore,
};
std::vector<ChunkedFile> ChunkedFiles;
- auto ChunkFile = [AttachmentTempPath](const IoHash& RawHash,
- IoBuffer& RawData,
- const IoBufferFileReference& FileRef,
- JobContext*) -> ChunkedFile {
+ auto ChunkFile = [](const IoHash& RawHash, IoBuffer& RawData, const IoBufferFileReference& FileRef, JobContext*) -> ChunkedFile {
ChunkedFile Chunked;
Stopwatch Timer;
@@ -1578,22 +1577,7 @@ BuildContainer(CidStore& ChunkStore,
std::filesystem::path AttachmentPath = AttachmentTempPath;
AttachmentPath.append(RawHash.ToHexString());
- uint32_t RetriesLeft = 3;
-
- IoBuffer TempAttachmentBuffer =
- WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath, [&](std::error_code& Ec) {
- if (RetriesLeft == 0)
- {
- return false;
- }
- ZEN_WARN("Failed to create temporary attachment '{}': '{}', retries left: {}.",
- AttachmentPath,
- Ec.message(),
- RetriesLeft);
- Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms
- RetriesLeft--;
- return true;
- });
+ IoBuffer TempAttachmentBuffer = WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath);
ZEN_INFO("Saved temp attachment to '{}', {} ({})",
AttachmentPath,
NiceBytes(RawSize),
@@ -1612,34 +1596,21 @@ BuildContainer(CidStore& ChunkStore,
std::filesystem::path AttachmentPath = AttachmentTempPath;
AttachmentPath.append(RawHash.ToHexString());
- uint32_t RetriesLeft = 3;
- IoBuffer TempAttachmentBuffer =
- WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath, [&](std::error_code& Ec) {
- if (RetriesLeft == 0)
- {
- return false;
- }
- ZEN_WARN("Failed to create temporary attachment '{}': '{}', retries left: {}.",
- AttachmentPath,
- Ec.message(),
- RetriesLeft);
- Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms
- RetriesLeft--;
- return true;
- });
+ uint64_t CompressedSize = Compressed.GetCompressedSize();
+ IoBuffer TempAttachmentBuffer = WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath);
ZEN_INFO("Saved temp attachment to '{}', {} ({})",
AttachmentPath,
NiceBytes(RawSize),
NiceBytes(TempAttachmentBuffer.GetSize()));
- if (Compressed.GetCompressedSize() > MaxChunkEmbedSize)
+ if (CompressedSize > MaxChunkEmbedSize)
{
OnLargeAttachment(RawHash, [Data = std::move(TempAttachmentBuffer)](const IoHash&) { return Data; });
ResolveLock.WithExclusiveLock([RawHash, &LargeChunkHashes]() { LargeChunkHashes.insert(RawHash); });
}
else
{
- UploadAttachment->Size = Compressed.GetCompressedSize();
+ UploadAttachment->Size = CompressedSize;
ResolveLock.WithExclusiveLock(
[RawHash, RawSize, &LooseUploadAttachments, Data = std::move(TempAttachmentBuffer)]() {
LooseUploadAttachments.insert_or_assign(RawHash, std::make_pair(RawSize, std::move(Data)));
@@ -1815,8 +1786,8 @@ BuildContainer(CidStore& ChunkStore,
std::vector<size_t> ReusedBlockFromChunking = FindReuseBlocks(KnownBlocks, ChunkedHashes, OptionalContext);
for (size_t KnownBlockIndex : ReusedBlockIndexes)
{
- const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
- for (const IoHash& KnownHash : KnownBlock.ChunkHashes)
+ const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
+ for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes)
{
if (ChunkedHashes.erase(KnownHash) == 1)
{
@@ -1834,7 +1805,7 @@ BuildContainer(CidStore& ChunkStore,
Blocks.reserve(ReuseBlockCount);
for (auto It = ReusedBlockIndexes.begin(); It != UniqueKnownBlocksEnd; It++)
{
- Blocks.push_back(KnownBlocks[*It]);
+ Blocks.push_back({KnownBlocks[*It]});
}
remotestore_impl::ReportMessage(OptionalContext,
fmt::format("Reused {} attachments from {} blocks", ReusedAttachmentCount, ReuseBlockCount));
@@ -1950,9 +1921,9 @@ BuildContainer(CidStore& ChunkStore,
{
// We can share the lock as we are not resizing the vector and only touch BlockHash at our own index
RwLock::SharedLockScope _(BlocksLock);
- Blocks[BlockIndex].ChunkHashes.insert(Blocks[BlockIndex].ChunkHashes.end(),
- BlockAttachmentHashes.begin(),
- BlockAttachmentHashes.end());
+ Blocks[BlockIndex].ChunkRawHashes.insert(Blocks[BlockIndex].ChunkRawHashes.end(),
+ BlockAttachmentHashes.begin(),
+ BlockAttachmentHashes.end());
}
uint64_t NowMS = Timer.GetElapsedTimeMs();
ZEN_INFO("Assembled block {} with {} chunks in {} ({})",
@@ -2198,7 +2169,7 @@ BuildContainer(CidStore& ChunkStore,
{
for (const ChunkBlockDescription& B : Blocks)
{
- ZEN_ASSERT(!B.ChunkHashes.empty());
+ ZEN_ASSERT(!B.ChunkRawHashes.empty());
if (BuildBlocks)
{
ZEN_ASSERT(B.BlockHash != IoHash::Zero);
@@ -2208,7 +2179,7 @@ BuildContainer(CidStore& ChunkStore,
OplogContinerWriter.AddBinaryAttachment("rawhash"sv, B.BlockHash);
OplogContinerWriter.BeginArray("chunks"sv);
{
- for (const IoHash& RawHash : B.ChunkHashes)
+ for (const IoHash& RawHash : B.ChunkRawHashes)
{
OplogContinerWriter.AddHash(RawHash);
}
@@ -2224,7 +2195,7 @@ BuildContainer(CidStore& ChunkStore,
{
OplogContinerWriter.BeginArray("chunks"sv);
{
- for (const IoHash& RawHash : B.ChunkHashes)
+ for (const IoHash& RawHash : B.ChunkRawHashes)
{
OplogContinerWriter.AddBinaryAttachment(RawHash);
}
@@ -2362,17 +2333,7 @@ SaveOplog(CidStore& ChunkStore,
BlockPath.append(Block.BlockHash.ToHexString());
try
{
- uint32_t RetriesLeft = 3;
- IoBuffer BlockBuffer = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), BlockPath, [&](std::error_code& Ec) {
- if (RetriesLeft == 0)
- {
- return false;
- }
- ZEN_WARN("Failed to create temporary oplog block '{}': '{}', retries left: {}.", BlockPath, Ec.message(), RetriesLeft);
- Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms
- RetriesLeft--;
- return true;
- });
+ IoBuffer BlockBuffer = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), BlockPath);
RwLock::ExclusiveLockScope __(AttachmentsLock);
CreatedBlocks.insert({Block.BlockHash, {.Payload = std::move(BlockBuffer), .Block = std::move(Block)}});
ZEN_DEBUG("Saved temp block to '{}', {}", AttachmentTempPath, NiceBytes(BlockBuffer.GetSize()));
@@ -2430,7 +2391,7 @@ SaveOplog(CidStore& ChunkStore,
OnBlock = UploadBlock;
}
- std::vector<ChunkBlockDescription> KnownBlocks;
+ std::vector<ThinChunkBlockDescription> KnownBlocks;
uint64_t TransferWallTimeMS = 0;
diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h
index 1ef0416b7..1210afc7c 100644
--- a/src/zenserver/projectstore/remoteprojectstore.h
+++ b/src/zenserver/projectstore/remoteprojectstore.h
@@ -66,7 +66,7 @@ public:
struct GetKnownBlocksResult : public Result
{
- std::vector<ChunkBlockDescription> Blocks;
+ std::vector<ThinChunkBlockDescription> Blocks;
};
struct RemoteStoreInfo
@@ -166,7 +166,7 @@ RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore,
bool CleanOplog,
JobContext* OptionalContext);
-std::vector<IoHash> GetBlockHashesFromOplog(CbObjectView ContainerObject);
-std::vector<ChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes);
+std::vector<IoHash> GetBlockHashesFromOplog(CbObjectView ContainerObject);
+std::vector<ThinChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes);
} // namespace zen
diff --git a/src/zenserver/windows/service.cpp b/src/zenserver/windows/service.cpp
index cb87df1f6..4be5e6205 100644
--- a/src/zenserver/windows/service.cpp
+++ b/src/zenserver/windows/service.cpp
@@ -21,7 +21,6 @@ HANDLE ghSvcStopEvent = NULL;
void SvcInstall(void);
-void ReportSvcStatus(DWORD, DWORD, DWORD);
void SvcReportEvent(LPTSTR);
WindowsService::WindowsService()
@@ -222,14 +221,7 @@ WindowsService::SvcMain()
return 1;
}
- // Report running status when initialization is complete.
-
- ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0);
-
int ReturnCode = Run();
-
- ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0);
-
return ReturnCode;
}
diff --git a/src/zenserver/windows/service.h b/src/zenserver/windows/service.h
index 7c9610983..ca0270a36 100644
--- a/src/zenserver/windows/service.h
+++ b/src/zenserver/windows/service.h
@@ -2,6 +2,8 @@
#pragma once
+#include <zencore/windows.h>
+
class WindowsService
{
public:
@@ -18,3 +20,5 @@ public:
int SvcMain();
static void __stdcall SvcCtrlHandler(unsigned long);
};
+
+VOID ReportSvcStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode, DWORD dwWaitHint);
diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp
index 905ba5ab2..8a4b977ad 100644
--- a/src/zenserver/workspaces/httpworkspaces.cpp
+++ b/src/zenserver/workspaces/httpworkspaces.cpp
@@ -589,7 +589,7 @@ void
HttpWorkspacesService::ShareAliasFilesRequest(HttpRouterRequest& Req)
{
HttpServerRequest& ServerRequest = Req.ServerRequest();
- std::string Alias = Req.GetCapture(1);
+ std::string_view Alias = Req.GetCapture(1);
if (Alias.empty())
{
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
@@ -608,7 +608,7 @@ void
HttpWorkspacesService::ShareAliasChunkInfoRequest(HttpRouterRequest& Req)
{
HttpServerRequest& ServerRequest = Req.ServerRequest();
- std::string Alias = Req.GetCapture(1);
+ std::string_view Alias = Req.GetCapture(1);
if (Alias.empty())
{
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
@@ -635,7 +635,7 @@ void
HttpWorkspacesService::ShareAliasBatchRequest(HttpRouterRequest& Req)
{
HttpServerRequest& ServerRequest = Req.ServerRequest();
- std::string Alias = Req.GetCapture(1);
+ std::string_view Alias = Req.GetCapture(1);
if (Alias.empty())
{
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
@@ -654,7 +654,7 @@ void
HttpWorkspacesService::ShareAliasEntriesRequest(HttpRouterRequest& Req)
{
HttpServerRequest& ServerRequest = Req.ServerRequest();
- std::string Alias = Req.GetCapture(1);
+ std::string_view Alias = Req.GetCapture(1);
if (Alias.empty())
{
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
@@ -673,7 +673,7 @@ void
HttpWorkspacesService::ShareAliasChunkRequest(HttpRouterRequest& Req)
{
HttpServerRequest& ServerRequest = Req.ServerRequest();
- std::string Alias = Req.GetCapture(1);
+ std::string_view Alias = Req.GetCapture(1);
if (Alias.empty())
{
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
@@ -700,7 +700,7 @@ void
HttpWorkspacesService::ShareAliasRequest(HttpRouterRequest& Req)
{
HttpServerRequest& ServerRequest = Req.ServerRequest();
- std::string Alias = Req.GetCapture(1);
+ std::string_view Alias = Req.GetCapture(1);
if (Alias.empty())
{
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-23 19:21:11 +0000