Merge branch 'main' into zs/web-ui-improvements

13 files changed, 128 insertions, 38 deletions

diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp
index 173f56386..0f9ef0287 100644
--- a/src/zenserver/compute/computeserver.cpp
+++ b/src/zenserver/compute/computeserver.cpp
@@ -82,7 +82,7 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ
 	ZEN_TRACE_CPU("ZenComputeServer::Initialize");
 	ZEN_MEMSCOPE(GetZenserverTag());
 
-	ZEN_INFO(ZEN_APP_NAME " initializing in HUB server mode");
+	ZEN_INFO(ZEN_APP_NAME " initializing in COMPUTE server mode");
 
 	const int EffectiveBasePort = ZenServerBase::Initialize(ServerConfig, ServerEntry);
 	if (EffectiveBasePort < 0)
@@ -91,7 +91,7 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ
 	}
 
 	// This is a workaround to make sure we can have automated tests. Without
-	// this the ranges for different child zen hub processes could overlap with
+	// this the ranges for different child zen compute processes could overlap with
 	// the main test range.
 	ZenServerEnvironment::SetBaseChildId(1000);
 
@@ -109,7 +109,7 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ
 void
 ZenComputeServer::Cleanup()
 {
-	ZEN_TRACE_CPU("ZenStorageServer::Cleanup");
+	ZEN_TRACE_CPU("ZenComputeServer::Cleanup");
 	ZEN_INFO(ZEN_APP_NAME " cleaning up");
 	try
 	{
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 2b157581f..579a65c5a 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -38,7 +38,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpSt
 #if ZEN_EMBED_HTML_ZIP
 	// Load an embedded Zip archive
 	IoBuffer HtmlZipDataBuffer(IoBuffer::Wrap, gHtmlZipData, sizeof(gHtmlZipData) - 1);
-	m_ZipFs = ZipFs(std::move(HtmlZipDataBuffer));
+	m_ZipFs = std::make_unique<ZipFs>(std::move(HtmlZipDataBuffer));
 #endif
 
 	if (m_Directory.empty() && !m_ZipFs)
@@ -114,6 +114,8 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 {
 	using namespace std::literals;
 
+	ExtendableStringBuilder<256> UriBuilder;
+
 	std::string_view Uri = Request.RelativeUriWithExtension();
 	for (; Uri.length() > 0 && Uri[0] == '/'; Uri = Uri.substr(1))
 		;
@@ -121,6 +123,11 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 	{
 		Uri = "index.html"sv;
 	}
+	else if (Uri.back() == '/')
+	{
+		UriBuilder << Uri << "index.html"sv;
+		Uri = UriBuilder;
+	}
 
 	// Dismiss if the URI contains .. anywhere to prevent arbitrary file reads
 	if (Uri.find("..") != Uri.npos)
@@ -145,24 +152,47 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 		return Request.WriteResponse(HttpResponseCode::Forbidden);
 	}
 
-	// The given content directory overrides any zip-fs discovered in the binary
-	if (!m_Directory.empty())
-	{
-		auto		 FullPath = m_Directory / std::filesystem::path(Uri).make_preferred();
-		FileContents File	  = ReadFile(FullPath);
+	auto WriteResponseForUri = [this,
+								&Request](std::string_view InUri, HttpResponseCode ResponseCode, HttpContentType ContentType) -> bool {
+		// The given content directory overrides any zip-fs discovered in the binary
+		if (!m_Directory.empty())
+		{
+			auto		 FullPath = m_Directory / std::filesystem::path(InUri).make_preferred();
+			FileContents File	  = ReadFile(FullPath);
 
-		if (!File.ErrorCode)
+			if (!File.ErrorCode)
+			{
+				Request.WriteResponse(ResponseCode, ContentType, File.Data[0]);
+
+				return true;
+			}
+		}
+
+		if (m_ZipFs)
 		{
-			return Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]);
+			if (IoBuffer FileBuffer = m_ZipFs->GetFile(InUri))
+			{
+				Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
+
+				return true;
+			}
 		}
-	}
 
-	if (IoBuffer FileBuffer = m_ZipFs.GetFile(Uri))
+		return false;
+	};
+
+	if (WriteResponseForUri(Uri, HttpResponseCode::OK, ContentType))
 	{
-		return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
+		return;
+	}
+	else if (WriteResponseForUri("404.html"sv, HttpResponseCode::NotFound, HttpContentType::kHTML))
+	{
+		return;
+	}
+	else
+	{
+		Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
 	}
-
-	Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
 }
 
 }  // namespace zen
diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h
index 84ffaac42..6d8585b72 100644
--- a/src/zenserver/frontend/frontend.h
+++ b/src/zenserver/frontend/frontend.h
@@ -7,6 +7,7 @@
 #include "zipfs.h"
 
 #include <filesystem>
+#include <memory>
 
 namespace zen {
 
@@ -20,9 +21,9 @@ public:
 	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
-	ZipFs				  m_ZipFs;
-	std::filesystem::path m_Directory;
-	HttpStatusService&	  m_StatusService;
+	std::unique_ptr<ZipFs> m_ZipFs;
+	std::filesystem::path  m_Directory;
+	HttpStatusService&	   m_StatusService;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index 4767029c0..3d90c18a8 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
diff --git a/src/zenserver/frontend/zipfs.cpp b/src/zenserver/frontend/zipfs.cpp
index f9c2bc8ff..42df0520f 100644
--- a/src/zenserver/frontend/zipfs.cpp
+++ b/src/zenserver/frontend/zipfs.cpp
@@ -149,13 +149,25 @@ ZipFs::ZipFs(IoBuffer&& Buffer)
 IoBuffer
 ZipFs::GetFile(const std::string_view& FileName) const
 {
-	FileMap::iterator Iter = m_Files.find(FileName);
-	if (Iter == m_Files.end())
 	{
-		return {};
+		RwLock::SharedLockScope _(m_FilesLock);
+
+		FileMap::const_iterator Iter = m_Files.find(FileName);
+		if (Iter == m_Files.end())
+		{
+			return {};
+		}
+
+		const FileItem& Item = Iter->second;
+		if (Item.GetSize() > 0)
+		{
+			return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize());
+		}
 	}
 
-	FileItem& Item = Iter->second;
+	RwLock::ExclusiveLockScope _(m_FilesLock);
+
+	FileItem& Item = m_Files.find(FileName)->second;
 	if (Item.GetSize() > 0)
 	{
 		return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize());
diff --git a/src/zenserver/frontend/zipfs.h b/src/zenserver/frontend/zipfs.h
index 1fa7da451..645121693 100644
--- a/src/zenserver/frontend/zipfs.h
+++ b/src/zenserver/frontend/zipfs.h
@@ -3,23 +3,23 @@
 #pragma once
 
 #include <zencore/iobuffer.h>
+#include <zencore/thread.h>
 
 #include <unordered_map>
 
 namespace zen {
 
-//////////////////////////////////////////////////////////////////////////
 class ZipFs
 {
 public:
-	ZipFs() = default;
-	ZipFs(IoBuffer&& Buffer);
+	explicit ZipFs(IoBuffer&& Buffer);
+
 	IoBuffer GetFile(const std::string_view& FileName) const;
-	inline	 operator bool() const { return !m_Files.empty(); }
 
 private:
 	using FileItem = MemoryView;
 	using FileMap  = std::unordered_map<std::string_view, FileItem>;
+	mutable RwLock m_FilesLock;
 	FileMap mutable m_Files;
 	IoBuffer m_Buffer;
 };
diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp
index 4d9da3a57..a00446a75 100644
--- a/src/zenserver/hub/hubservice.cpp
+++ b/src/zenserver/hub/hubservice.cpp
@@ -151,6 +151,7 @@ struct StorageServerInstance
 	inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); }
 
 private:
+	void				  WakeLocked();
 	RwLock				  m_Lock;
 	std::string			  m_ModuleId;
 	std::atomic<bool>	  m_IsProvisioned{false};
@@ -211,7 +212,7 @@ StorageServerInstance::Provision()
 
 	if (m_IsHibernated)
 	{
-		Wake();
+		WakeLocked();
 	}
 	else
 	{
@@ -294,9 +295,14 @@ StorageServerInstance::Hibernate()
 void
 StorageServerInstance::Wake()
 {
-	// Start server in-place using existing data
-
 	RwLock::ExclusiveLockScope _(m_Lock);
+	WakeLocked();
+}
+
+void
+StorageServerInstance::WakeLocked()
+{
+	// Start server in-place using existing data
 
 	if (!m_IsHibernated)
 	{
diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index 7a4ba951d..d0a0db417 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -105,7 +105,7 @@ ZenHubServer::Initialize(const ZenHubServerConfig& ServerConfig, ZenServerState:
 void
 ZenHubServer::Cleanup()
 {
-	ZEN_TRACE_CPU("ZenStorageServer::Cleanup");
+	ZEN_TRACE_CPU("ZenHubServer::Cleanup");
 	ZEN_INFO(ZEN_APP_NAME " cleaning up");
 	try
 	{
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index ee783d2a6..571dd3b4f 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -267,6 +267,14 @@ main(int argc, char* argv[])
 	using namespace zen;
 	using namespace std::literals;
 
+	// note: doctest has locally (in thirdparty) been fixed to not cause shutdown
+	// 		 crashes due to TLS destructors
+	//
+	// mimalloc on the other hand might still be causing issues, in which case
+	// we should work out either how to eliminate the mimalloc dependency or how
+	// to configure it in a way that doesn't cause shutdown issues
+
+#if 0 	
 	auto _ = zen::MakeGuard([] {
 		// Allow some time for worker threads to unravel, in an effort
 		// to prevent shutdown races in TLS object destruction, mainly due to
@@ -277,6 +285,7 @@ main(int argc, char* argv[])
 		// shutdown crashes observed in some situations.
 		WaitForThreads(1000);
 	});
+#endif
 
 	enum
 	{
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index f5ba30616..bf7afcc02 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -185,7 +185,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
 	{
 		const HttpRange& Range		 = Ranges.front();
 		const uint64_t	 BlobSize	 = Blob.GetSize();
-		const uint64_t	 MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0;
+		const uint64_t	 MaxBlobSize = Range.Start < BlobSize ? BlobSize - Range.Start : 0;
 		const uint64_t	 RangeSize	 = Min(Range.End - Range.Start + 1, MaxBlobSize);
 		if (Range.Start + RangeSize > BlobSize)
 		{
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 416e2ed69..2b5474d00 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -244,6 +244,8 @@ namespace {
 	{
 		std::shared_ptr<RemoteProjectStore> Store;
 		std::string							Description;
+		double								HostLatencySec	= -1.0;
+		double								CacheLatencySec = -1.0;
 	};
 
 	CreateRemoteStoreResult CreateRemoteStore(LoggerRef					   InLog,
@@ -261,6 +263,8 @@ namespace {
 		using namespace std::literals;
 
 		std::shared_ptr<RemoteProjectStore> RemoteStore;
+		double								HostLatencySec	= -1.0;
+		double								CacheLatencySec = -1.0;
 
 		if (CbObjectView File = Params["file"sv].AsObjectView(); File)
 		{
@@ -495,7 +499,9 @@ namespace {
 														 /*Quiet*/ false,
 														 /*Unattended*/ false,
 														 /*Hidden*/ true,
-														 GetTinyWorkerPool(EWorkloadType::Background));
+														 GetTinyWorkerPool(EWorkloadType::Background),
+														 HostLatencySec,
+														 CacheLatencySec);
 		}
 
 		if (!RemoteStore)
@@ -503,7 +509,10 @@ namespace {
 			return {nullptr, "Unknown remote store type"};
 		}
 
-		return {std::move(RemoteStore), ""};
+		return CreateRemoteStoreResult{.Store			= std::move(RemoteStore),
+									   .Description		= "",
+									   .HostLatencySec	= HostLatencySec,
+									   .CacheLatencySec = CacheLatencySec};
 	}
 
 	std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result)
@@ -2356,15 +2365,19 @@ HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req)
 		tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
 
 		auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); };
-		auto OnNeedBlock   = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) {
+		auto OnNeedBlock   = [&AttachmentsLock, &Attachments](ThinChunkBlockDescription&& ThinBlockDescription,
+															  std::vector<uint32_t>&&	  NeededChunkIndexes) {
 			  RwLock::ExclusiveLockScope _(AttachmentsLock);
-			  if (BlockHash != IoHash::Zero)
+			  if (ThinBlockDescription.BlockHash != IoHash::Zero)
 			  {
-				  Attachments.insert(BlockHash);
+				  Attachments.insert(ThinBlockDescription.BlockHash);
 			  }
 			  else
 			  {
-				  Attachments.insert(ChunkHashes.begin(), ChunkHashes.end());
+				  for (uint32_t ChunkIndex : NeededChunkIndexes)
+				  {
+					  Attachments.insert(ThinBlockDescription.ChunkRawHashes[ChunkIndex]);
+				  }
 			  }
 		};
 		auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) {
@@ -2663,6 +2676,8 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 				bool		 CleanOplog				  = Params["clean"].AsBool(false);
 				bool		 BoostWorkerCount		  = Params["boostworkercount"].AsBool(false);
 				bool		 BoostWorkerMemory		  = Params["boostworkermemory"sv].AsBool(false);
+				EPartialBlockRequestMode PartialBlockRequestMode =
+					PartialBlockRequestModeFromString(Params["partialblockrequestmode"sv].AsString("true"));
 
 				CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Log(),
 																			  Params,
@@ -2688,6 +2703,9 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 					 Force,
 					 IgnoreMissingAttachments,
 					 CleanOplog,
+					 PartialBlockRequestMode,
+					 HostLatencySec	 = RemoteStoreResult.HostLatencySec,
+					 CacheLatencySec = RemoteStoreResult.CacheLatencySec,
 					 BoostWorkerCount](JobContext& Context) {
 						Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}",
 														  Oplog->GetOuterProjectIdentifier(),
@@ -2709,6 +2727,9 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 																		Force,
 																		IgnoreMissingAttachments,
 																		CleanOplog,
+																		PartialBlockRequestMode,
+																		HostLatencySec,
+																		CacheLatencySec,
 																		&Context);
 						auto					   Response = ConvertResult(Result);
 						ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index ff854b72d..3d81db656 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -700,6 +700,15 @@ ZenStorageServer::Run()
 
 	ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId());
 
+	if (m_FrontendService)
+	{
+		ZEN_INFO("frontend link: {}", m_Http->GetServiceUri(m_FrontendService.get()));
+	}
+	else
+	{
+		ZEN_INFO("frontend service disabled");
+	}
+
 #if ZEN_PLATFORM_WINDOWS
 	if (zen::windows::IsRunningOnWine())
 	{
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 7bf6126df..5fd35d9b4 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -617,6 +617,8 @@ ZenServerMain::Run()
 			{
 				ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message());
 				Sleep(500);
+
+				m_LockFile.Create(LockFilePath, MakeLockData(false), Ec);
 				if (Ec)
 				{
 					ZEN_WARN(ZEN_APP_NAME " exiting, unable to grab lock at '{}' (reason: '{}')", LockFilePath, Ec.message());