use 32 bit offset and size in BlockStoreLocation (#581)

- Improvement: Reduce memory usage in GC and diskbucket flush

1 files changed, 23 insertions, 18 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index cc727787f..918f464ac 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -384,7 +384,7 @@ BlockStore::GetFreeBlockIndex(uint32_t ProbeIndex, RwLock::ExclusiveLockScope&,
 }
 
 void
-BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, const WriteChunkCallback& Callback)
+BlockStore::WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback)
 {
 	ZEN_TRACE_CPU("BlockStore::WriteChunk");
 
@@ -393,12 +393,14 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, cons
 	ZEN_ASSERT(Size <= m_MaxBlockSize);
 	ZEN_ASSERT(Alignment > 0u);
 
+	uint32_t ChunkSize = gsl::narrow<uint32_t>(Size);
+
 	RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
 
 	uint32_t WriteBlockIndex	 = m_WriteBlockIndex.load(std::memory_order_acquire);
 	bool	 IsWriting			 = !!m_WriteBlock;
-	uint64_t AlignedInsertOffset = RoundUp(m_CurrentInsertOffset, Alignment);
-	if (!IsWriting || (AlignedInsertOffset + Size) > m_MaxBlockSize)
+	uint32_t AlignedInsertOffset = RoundUp(m_CurrentInsertOffset, Alignment);
+	if (!IsWriting || (AlignedInsertOffset + ChunkSize) > m_MaxBlockSize)
 	{
 		if (m_WriteBlock)
 		{
@@ -423,16 +425,16 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, cons
 		m_CurrentInsertOffset = 0;
 		AlignedInsertOffset	  = 0;
 	}
-	uint64_t AlignedWriteSize	   = AlignedInsertOffset - m_CurrentInsertOffset + Size;
-	m_CurrentInsertOffset		   = AlignedInsertOffset + Size;
+	uint32_t AlignedWriteSize	   = AlignedInsertOffset - m_CurrentInsertOffset + ChunkSize;
+	m_CurrentInsertOffset		   = AlignedInsertOffset + ChunkSize;
 	Ref<BlockStoreFile> WriteBlock = m_WriteBlock;
 	m_ActiveWriteBlocks.push_back(WriteBlockIndex);
 	InsertLock.ReleaseNow();
 
-	WriteBlock->Write(Data, Size, AlignedInsertOffset);
+	WriteBlock->Write(Data, ChunkSize, AlignedInsertOffset);
 	m_TotalSize.fetch_add(AlignedWriteSize, std::memory_order::relaxed);
 
-	Callback({.BlockIndex = WriteBlockIndex, .Offset = AlignedInsertOffset, .Size = Size});
+	Callback({.BlockIndex = WriteBlockIndex, .Offset = AlignedInsertOffset, .Size = ChunkSize});
 
 	{
 		RwLock::ExclusiveLockScope _(m_InsertLock);
@@ -505,7 +507,7 @@ void
 BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 						 const std::vector<BlockStoreLocation>& ChunkLocations,
 						 const ChunkIndexArray&					KeepChunkIndexes,
-						 uint64_t								PayloadAlignment,
+						 uint32_t								PayloadAlignment,
 						 bool									DryRun,
 						 const ReclaimCallback&					ChangeCallback,
 						 const ClaimDiskReserveCallback&		DiskReserveCallback)
@@ -754,9 +756,9 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 				{
 					const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
 					Chunk.resize(ChunkLocation.Size);
-					OldBlockFile->Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
+					OldBlockFile->Read(Chunk.data(), ChunkLocation.Size, ChunkLocation.Offset);
 
-					if (!NewBlockFile || (WriteOffset + Chunk.size() > m_MaxBlockSize))
+					if (!NewBlockFile || (WriteOffset + ChunkLocation.Size > m_MaxBlockSize))
 					{
 						uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order_relaxed);
 
@@ -830,10 +832,12 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 						WriteOffset	  = 0;
 					}
 
-					NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
-					MovedChunks.push_back({ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = WriteOffset, .Size = Chunk.size()}});
+					NewBlockFile->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
+					MovedChunks.push_back(
+						{ChunkIndex,
+						 {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
 					uint64_t OldOffset = WriteOffset;
-					WriteOffset		   = RoundUp(WriteOffset + Chunk.size(), PayloadAlignment);
+					WriteOffset		   = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
 					m_TotalSize.fetch_add(WriteOffset - OldOffset, std::memory_order::relaxed);
 				}
 				Chunk.clear();
@@ -1033,7 +1037,7 @@ BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
 
 void
 BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
-						  uint64_t						  PayloadAlignment,
+						  uint32_t						  PayloadAlignment,
 						  const CompactCallback&		  ChangeCallback,
 						  const ClaimDiskReserveCallback& DiskReserveCallback)
 {
@@ -1218,9 +1222,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 				WriteOffset	  = 0;
 			}
 
-			NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
-			MovedChunks.push_back({ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = WriteOffset, .Size = Chunk.size()}});
-			WriteOffset = RoundUp(WriteOffset + Chunk.size(), PayloadAlignment);
+			NewBlockFile->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
+			MovedChunks.push_back(
+				{ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
+			WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
 			AddedSize += Chunk.size();
 		}
 		Chunk.clear();
@@ -1403,7 +1408,7 @@ TEST_CASE("blockstore.blockfile")
 }
 
 namespace blockstore::impl {
-	BlockStoreLocation WriteStringAsChunk(BlockStore& Store, std::string_view String, size_t PayloadAlignment)
+	BlockStoreLocation WriteStringAsChunk(BlockStore& Store, std::string_view String, uint32_t PayloadAlignment)
 	{
 		BlockStoreLocation Location;
 		Store.WriteChunk(String.data(), String.length(), PayloadAlignment, [&](const BlockStoreLocation& L) { Location = L; });