From 96f44f2f2d8cbcda254d0b193f5a1aece645daeb Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Mon, 22 Apr 2024 20:21:02 +0200
Subject: InsertChunks for CAS store (#55)

- Improvement: Add batching when writing multiple small chunks to block store - decreases I/O load significantly on oplog import
---
 src/zenstore/include/zenstore/blockstore.h |  3 +++
 src/zenstore/include/zenstore/cidstore.h   | 21 ++++++++++++---------
 2 files changed, 15 insertions(+), 9 deletions(-)

(limited to 'src/zenstore/include')

diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index 656040936..c28aa8102 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -161,6 +161,9 @@ public:
 
 	void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback);
 
+	typedef std::function<void(std::span<BlockStoreLocation> Locations)> WriteChunksCallback;
+	void WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback);
+
 	IoBuffer TryGetChunk(const BlockStoreLocation& Location) const;
 	void	 Flush(bool ForceNewBlock);
 
diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h
index 4c9f30608..54f562767 100644
--- a/src/zenstore/include/zenstore/cidstore.h
+++ b/src/zenstore/include/zenstore/cidstore.h
@@ -73,15 +73,18 @@ public:
 		kMayBeMovedInPlace
 	};
 
-	void			 Initialize(const CidStoreConfiguration& Config);
-	InsertResult	 AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace);
-	virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override;
-	bool			 ContainsChunk(const IoHash& DecompressedId);
-	void			 FilterChunks(HashKeySet& InOutChunks);
-	void			 Flush();
-	void			 ScrubStorage(ScrubContext& Ctx);
-	CidStoreSize	 TotalSize() const;
-	CidStoreStats	 Stats() const;
+	void					  Initialize(const CidStoreConfiguration& Config);
+	InsertResult			  AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace);
+	std::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas,
+										std::span<IoHash>	RawHashes,
+										InsertMode			Mode = InsertMode::kMayBeMovedInPlace);
+	virtual IoBuffer		  FindChunkByCid(const IoHash& DecompressedId) override;
+	bool					  ContainsChunk(const IoHash& DecompressedId);
+	void					  FilterChunks(HashKeySet& InOutChunks);
+	void					  Flush();
+	void					  ScrubStorage(ScrubContext& Ctx);
+	CidStoreSize			  TotalSize() const;
+	CidStoreStats			  Stats() const;
 
 	virtual void ReportMetrics(StatsMetrics& Statsd) override;
 
-- 
cgit v1.2.3