jupiter builds stats upload (#315)

 - Improvement: At end of build upload we post statistics to the Jupiter build stats endpoint:
	- `totalSize`
	- `reusedRatio`
	- `reusedBlockCount`
	- `reusedBlockByteCount`
	- `newBlockCount`
	- `newBlockByteCount`
	- `uploadedCount`
	- `uploadedByteCount`
	- `elapsedTimeSec`
	- `uploadedBytesPerSec`

6 files changed, 97 insertions, 15 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index a1fb1a94a..98031116a 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -3343,19 +3343,6 @@ namespace {
 			ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats);
 		}
 
-		struct ValidateStatistics
-		{
-			uint64_t			  BuildBlobSize				= 0;
-			uint64_t			  BuildPartSize				= 0;
-			uint64_t			  ChunkAttachmentCount		= 0;
-			uint64_t			  BlockAttachmentCount		= 0;
-			std::atomic<uint64_t> DownloadedAttachmentCount = 0;
-			std::atomic<uint64_t> VerifiedAttachmentCount	= 0;
-			std::atomic<uint64_t> DownloadedByteCount		= 0;
-			std::atomic<uint64_t> VerifiedByteCount			= 0;
-			uint64_t			  ElapsedWallTimeUS			= 0;
-		};
-
 		ZEN_CONSOLE_VERBOSE(
 			"Folder scanning stats:"
 			"\n    FoundFileCount:        {}"
@@ -3572,7 +3559,7 @@ namespace {
 
 			NiceBytes(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load()),
 			NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes),
-			NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes * 8))),
+			NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)),
 
 			UploadStats.BlockCount.load(),
 			NiceBytes(UploadStats.BlocksBytes.load()),
@@ -3582,6 +3569,21 @@ namespace {
 			MultipartAttachmentStats,
 
 			ValidateInfo);
+
+		Storage.PutBuildPartStats(
+			BuildId,
+			BuildPartId,
+			{{"totalSize", double(LocalFolderScanStats.FoundFileByteCount.load())},
+			 {"reusedRatio", AcceptedByteCountPercent / 100.0},
+			 {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)},
+			 {"reusedBlockByteCount", double(FindBlocksStats.AcceptedByteCount)},
+			 {"newBlockCount", double(FindBlocksStats.NewBlocksCount)},
+			 {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)},
+			 {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())},
+			 {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())},
+			 {"uploadedBytesPerSec",
+			  double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))},
+			 {"elapsedTimeSec", double(ProcessTimer.GetElapsedTimeMs() / 1000.0)}});
 	}
 
 	void VerifyFolder(const ChunkedFolderContent&  Content,
@@ -7966,7 +7968,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			if (m_BuildsUrl.empty() && StoragePath.empty())
 			{
-				m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").generic_string();
+				StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
 				CreateDirectories(StoragePath);
 				CleanDirectory(StoragePath, {});
 			}
diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index 47a4e1cc4..130fec355 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -502,6 +502,30 @@ public:
 		return Result;
 	}
 
+	virtual void PutBuildPartStats(const Oid&								  BuildId,
+								   const Oid&								  BuildPartId,
+								   const tsl::robin_map<std::string, double>& FloatStats) override
+	{
+		CbObjectWriter Request;
+		Request.BeginObject("floatStats"sv);
+		for (auto It : FloatStats)
+		{
+			Request.AddFloat(It.first, It.second);
+		}
+		Request.EndObject();
+		CbObject Payload = Request.Save();
+
+		SimulateLatency(Payload.GetSize(), 0);
+
+		const std::filesystem::path BuildPartStatsDataPath = GetBuildPartStatsPath(BuildId, BuildPartId);
+		CreateDirectories(BuildPartStatsDataPath.parent_path());
+
+		TemporaryFile::SafeWriteFile(BuildPartStatsDataPath, Payload.GetView());
+		WriteAsJson(BuildPartStatsDataPath, Payload);
+
+		SimulateLatency(0, 0);
+	}
+
 protected:
 	std::filesystem::path GetBuildsFolder() const { return m_StoragePath / "builds"; }
 	std::filesystem::path GetBlobsFolder() const { return m_StoragePath / "blobs"; }
@@ -520,6 +544,11 @@ protected:
 		return GetBuildPartFolder(BuildId, BuildPartId) / "metadata.cb";
 	}
 
+	std::filesystem::path GetBuildPartStatsPath(const Oid& BuildId, const Oid& BuildPartId) const
+	{
+		return GetBuildPartFolder(BuildId, BuildPartId) / fmt::format("stats_{}.cb", Oid::NewOid());
+	}
+
 	std::filesystem::path GetBlobPayloadPath(const IoHash& RawHash) const { return GetBlobsFolder() / fmt::format("{}.cbz", RawHash); }
 
 	std::filesystem::path GetBlobMetadataPath(const IoHash& RawHash) const
diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h
index 9d2bab170..2ebd65a00 100644
--- a/src/zenutil/include/zenutil/buildstorage.h
+++ b/src/zenutil/include/zenutil/buildstorage.h
@@ -5,6 +5,10 @@
 #include <zencore/compactbinary.h>
 #include <zenutil/chunkblock.h>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 namespace zen {
 
 class BuildStorage
@@ -53,6 +57,8 @@ public:
 	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData)				 = 0;
 	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId)											 = 0;
 	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0;
+
+	virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map<std::string, double>& FloatStats) = 0;
 };
 
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h
index 2c5fc73b8..fda4a7bfe 100644
--- a/src/zenutil/include/zenutil/jupiter/jupitersession.h
+++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h
@@ -155,6 +155,12 @@ public:
 	JupiterResult			FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
 	JupiterResult			GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload);
 
+	JupiterResult PutBuildPartStats(std::string_view Namespace,
+									std::string_view BucketId,
+									const Oid&		 BuildId,
+									const Oid&		 BuildPartId,
+									IoBuffer		 Payload);
+
 private:
 	inline LoggerRef Log() { return m_Log; }
 
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index bf89ce785..d70fd8c00 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -348,6 +348,30 @@ public:
 		return SortedBlockDescriptions;
 	}
 
+	virtual void PutBuildPartStats(const Oid&								  BuildId,
+								   const Oid&								  BuildPartId,
+								   const tsl::robin_map<std::string, double>& FloatStats) override
+	{
+		ZEN_UNUSED(BuildId, BuildPartId, FloatStats);
+		CbObjectWriter Request;
+		Request.BeginObject("floatStats"sv);
+		for (auto It : FloatStats)
+		{
+			Request.AddFloat(It.first, It.second);
+		}
+		Request.EndObject();
+		IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+		Payload.SetContentType(ZenContentType::kCbObject);
+		JupiterResult PutBuildPartStatsResult = m_Session.PutBuildPartStats(m_Namespace, m_Bucket, BuildId, BuildPartId, Payload);
+		AddStatistic(PutBuildPartStatsResult);
+		if (!PutBuildPartStatsResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed posting build part statistics: {} ({})",
+												 PutBuildPartStatsResult.Reason,
+												 PutBuildPartStatsResult.ErrorCode));
+		}
+	}
+
 private:
 	static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload)
 	{
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index 68f214c06..2e4fe5258 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -776,4 +776,19 @@ JupiterSession::GetBlockMetadata(std::string_view Namespace, std::string_view Bu
 	return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv);
 }
 
+JupiterResult
+JupiterSession::PutBuildPartStats(std::string_view Namespace,
+								  std::string_view BucketId,
+								  const Oid&	   BuildId,
+								  const Oid&	   BuildPartId,
+								  IoBuffer		   Payload)
+{
+	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
+	HttpClient::Response Response =
+		m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/stats", Namespace, BucketId, BuildId, BuildPartId),
+						 Payload,
+						 HttpClient::Accept(ZenContentType::kCbObject));
+	return detail::ConvertResponse(Response, "JupiterSession::PutBuildPartStats"sv);
+}
+
 }  // namespace zen