improved block gen logic (#302)

 - Improvement: Reduced memory usage during upload and part upload validation
 - Improvement: Reduced I/O usage during upload and download
 - Improvement: Faster block regeneration when uploading in response to PutBuild/FinalizeBuild
 - Improvement: More trace scopes for build upload operations
 - Bugfix: Fixed crash during download when trying to write outside a file range

1 files changed, 17 insertions, 0 deletions

diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index e57109006..47a4e1cc4 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -8,6 +8,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 
 namespace zen {
 
@@ -36,6 +37,7 @@ public:
 
 	virtual CbObject ListBuilds(CbObject Query) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::ListBuilds");
 		ZEN_UNUSED(Query);
 
 		SimulateLatency(Query.GetSize(), 0);
@@ -72,6 +74,7 @@ public:
 
 	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuild");
 		SimulateLatency(MetaData.GetSize(), 0);
 
 		Stopwatch ExecutionTimer;
@@ -93,6 +96,7 @@ public:
 
 	virtual CbObject GetBuild(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuild");
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
@@ -105,6 +109,7 @@ public:
 
 	virtual void FinalizeBuild(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuild");
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
@@ -119,6 +124,7 @@ public:
 																std::string_view PartName,
 																const CbObject&	 MetaData) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuildPart");
 		SimulateLatency(MetaData.GetSize(), 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
@@ -164,6 +170,7 @@ public:
 
 	virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildPart");
 		SimulateLatency(0, 0);
 
 		Stopwatch ExecutionTimer;
@@ -186,6 +193,7 @@ public:
 
 	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuildPart");
 		SimulateLatency(0, 0);
 
 		Stopwatch ExecutionTimer;
@@ -215,6 +223,7 @@ public:
 							  ZenContentType		 ContentType,
 							  const CompositeBuffer& Payload) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuildBlob");
 		ZEN_UNUSED(BuildId);
 		ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
 		SimulateLatency(Payload.GetSize(), 0);
@@ -242,6 +251,7 @@ public:
 																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
 																 std::function<void(uint64_t, bool)>&& OnSentBytes) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob");
 		ZEN_UNUSED(BuildId);
 		ZEN_UNUSED(ContentType);
 		SimulateLatency(0, 0);
@@ -281,6 +291,7 @@ public:
 				uint64_t Size = Min(32u * 1024u * 1024u, PayloadSize - Offset);
 
 				WorkItems.push_back([this, RawHash, BlockPath, Workload, Offset, Size]() {
+					ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob_Work");
 					IoBuffer PartPayload = Workload->Transmitter(Offset, Size);
 					SimulateLatency(PartPayload.GetSize(), 0);
 
@@ -327,6 +338,7 @@ public:
 
 	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlob");
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
@@ -363,6 +375,7 @@ public:
 		uint64_t																			   ChunkSize,
 		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob");
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
@@ -392,6 +405,7 @@ public:
 			{
 				uint64_t Size = Min(ChunkSize, BlobSize - Offset);
 				WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() {
+					ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob_Work");
 					SimulateLatency(0, 0);
 					IoBuffer PartPayload(Size);
 					Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset);
@@ -411,6 +425,7 @@ public:
 
 	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata");
 		ZEN_UNUSED(BuildId);
 
 		SimulateLatency(MetaData.GetSize(), 0);
@@ -429,6 +444,7 @@ public:
 
 	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::FindBlocks");
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
@@ -461,6 +477,7 @@ public:
 
 	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata");
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;