don't use mmap when reading attachment meta filesde/no-mmap-for-attachment-meta-files

7 files changed, 166 insertions, 71 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 4a943a565..c53d129e3 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -2086,13 +2086,18 @@ ProjectStore::Oplog::GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, b
 
 	if (StoreMetaDataOnDisk && m_MetaValid)
 	{
-		IoBuffer MetadataPayload = IoBufferBuilder::MakeFromFile(m_MetaPath);
-		if (MetadataPayload)
+		BasicFile		MetadataPayload;
+		std::error_code Ec;
+		if (MetadataPayload.Open(m_MetaPath, BasicFile::Mode::kRead, Ec); !Ec)
 		{
 			ZEN_TRACE_CPU("Store::Oplog::GetAttachmentsFromMetaData");
 			if (GetAttachmentsFromMetaData<Oid, IoHash>(
 					MetadataPayload,
 					OplogMetaDataExpectedMagic,
+					[&](std::uint32_t EntryCount, uint64_t AttachmentCount) {
+						ZEN_UNUSED(EntryCount);
+						OutAttachments.reserve(OutAttachments.size() + AttachmentCount);
+					},
 					[&](std::span<const Oid> Keys, std::span<const uint32_t> AttachmentCounts, std::span<const IoHash> Attachments) {
 						ZEN_UNUSED(Keys);
 						ZEN_UNUSED(AttachmentCounts);
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 592d1c7fb..79f4f89a1 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -210,15 +210,20 @@ IsMetaDataValid(const std::filesystem::path& BlockPath, const std::filesystem::p
 	return true;
 }
 
-IoBuffer
-BlockStoreFile::GetMetaData() const
+bool
+BlockStoreFile::GetMetaData(BasicFile& OutData) const
 {
 	const std::filesystem::path MetaPath = GetMetaPath();
 	if (IsMetaDataValid(m_Path, MetaPath))
 	{
-		return IoBufferBuilder::MakeFromFile(MetaPath);
+		std::error_code Ec;
+		OutData.Open(MetaPath, BasicFile::Mode::kRead, Ec);
+		if (!Ec)
+		{
+			return true;
+		}
 	}
-	return {};
+	return false;
 }
 
 uint64_t
@@ -1663,15 +1668,15 @@ BlockStore::SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload)
 	}
 }
 
-IoBuffer
-BlockStore::GetMetaData(uint32_t BlockIndex) const
+bool
+BlockStore::GetMetaData(uint32_t BlockIndex, BasicFile& OutData) const
 {
 	RwLock::SharedLockScope _(m_InsertLock);
 	if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end() && It->second)
 	{
-		return It->second->GetMetaData();
+		return It->second->GetMetaData(OutData);
 	}
-	return {};
+	return false;
 }
 
 #if ZEN_WITH_TESTS
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 928165185..0611b608c 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -3296,6 +3296,8 @@ public:
 			size_t					  InlineEntryCount = 0;
 			BlockStore::BlockUsageMap BlockUsage;
 			{
+				BlockUsage.reserve(m_Bucket.m_BlockStore.GetBlockCount());
+
 				RwLock::SharedLockScope ___(m_Bucket.m_IndexLock);
 				for (const auto& Entry : m_Bucket.m_Index)
 				{
@@ -3562,8 +3564,8 @@ ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t					 BlockI
 															std::vector<IoHash>&		 OutReferences) const
 {
 	ZEN_TRACE_CPU("Z$::Bucket::ReadAttachmentsFromMetaData");
-	IoBuffer MetaDataPayload = m_BlockStore.GetMetaData(BlockIndex);
-	if (MetaDataPayload)
+	BasicFile MetaDataPayload;
+	if (m_BlockStore.GetMetaData(BlockIndex, MetaDataPayload))
 	{
 		tsl::robin_set<IoHash, IoHash::Hasher> WantedKeys;
 		WantedKeys.reserve(ChunkIndexes.size());
@@ -3572,33 +3574,40 @@ ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t					 BlockI
 			WantedKeys.insert(InlineKeys[ChunkIndex]);
 		}
 		ZEN_TRACE_CPU("Z$::Bucket::GetAttachmentsFromMetaData");
-		return GetAttachmentsFromMetaData<IoHash, IoHash>(
-			MetaDataPayload,
-			BlockMetaDataExpectedMagic,
-			[&](std::span<const IoHash> Keys, std::span<const uint32_t> AttachmentCounts, std::span<const IoHash> Attachments) {
-				auto AttachmentReadIt = Attachments.begin();
-				OutReferences.resize(OutReferences.size() + Attachments.size());
-				auto OutReferencesWriteIt = OutReferences.end() - Attachments.size();
-				auto KeyIt				  = Keys.begin();
-				for (uint32_t AttachmentCount : AttachmentCounts)
-				{
-					if (AttachmentCount > 0)
+		auto OutReferencesWriteIt = OutReferences.end();
+		if (GetAttachmentsFromMetaData<IoHash, IoHash>(
+				MetaDataPayload,
+				BlockMetaDataExpectedMagic,
+				[&](std::uint32_t EntryCount, uint64_t AttachmentCount) {
+					ZEN_UNUSED(EntryCount);
+					OutReferences.resize(OutReferences.size() + AttachmentCount);
+					OutReferencesWriteIt = OutReferences.end() - AttachmentCount;
+				},
+				[&](std::span<const IoHash> Keys, std::span<const uint32_t> AttachmentCounts, std::span<const IoHash> Attachments) {
+					auto AttachmentReadIt = Attachments.begin();
+					auto KeyIt			  = Keys.begin();
+					for (uint32_t AttachmentCount : AttachmentCounts)
 					{
-						if (WantedKeys.contains(*KeyIt))
-						{
-							memcpy(&(*OutReferencesWriteIt), &(*AttachmentReadIt), sizeof(IoHash) * AttachmentCount);
-							OutReferencesWriteIt += AttachmentCount;
-							AttachmentReadIt += AttachmentCount;
-						}
-						else
+						if (AttachmentCount > 0)
 						{
-							AttachmentReadIt += AttachmentCount;
+							if (WantedKeys.contains(*KeyIt))
+							{
+								memcpy(&(*OutReferencesWriteIt), &(*AttachmentReadIt), sizeof(IoHash) * AttachmentCount);
+								OutReferencesWriteIt += AttachmentCount;
+								AttachmentReadIt += AttachmentCount;
+							}
+							else
+							{
+								AttachmentReadIt += AttachmentCount;
+							}
 						}
+						KeyIt++;
 					}
-					KeyIt++;
-				}
-				OutReferences.erase(OutReferencesWriteIt, OutReferences.end());
-			});
+				}))
+		{
+			OutReferences.erase(OutReferencesWriteIt, OutReferences.end());
+			return true;
+		}
 	}
 	return false;
 }
@@ -3622,24 +3631,23 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(GcCtx& Ctx, bool StateIsAlreadyLoc
 
 	std::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys;
 	{
+		size_t BlockCount = m_BlockStore.GetBlockCount();
+
 		std::vector<IoHash>					  InlineKeys;
 		std::vector<BlockStoreLocation>		  InlineLocations;
 		std::vector<std::vector<std::size_t>> InlineBlockChunkIndexes;
 
+		InlineBlockChunkIndexes.reserve(BlockCount);
 		{
-			std::unordered_map<uint32_t, std::size_t> BlockIndexToChunkIndexes;
-			std::unique_ptr<RwLock::SharedLockScope>  StateLock;
+			tsl::robin_map<uint32_t, std::size_t> BlockIndexToChunkIndexes;
+			BlockIndexToChunkIndexes.reserve(BlockCount);
+			std::unique_ptr<RwLock::SharedLockScope> StateLock;
 			if (!StateIsAlreadyLocked)
 			{
 				StateLock = std::make_unique<RwLock::SharedLockScope>(m_IndexLock);
 			}
 			for (const auto& Entry : m_Index)
 			{
-				if (Ctx.IsCancelledFlag.load())
-				{
-					return false;
-				}
-
 				PayloadIndex		 EntryIndex = Entry.second;
 				const BucketPayload& Payload	= m_Payloads[EntryIndex];
 				const DiskLocation&	 Loc		= Payload.Location;
@@ -3666,6 +3674,11 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(GcCtx& Ctx, bool StateIsAlreadyLoc
 				}
 				else
 				{
+					if (Ctx.IsCancelledFlag.load())
+					{
+						return false;
+					}
+
 					BlockIndexToChunkIndexes.insert_or_assign(ChunkLocation.BlockIndex, InlineBlockChunkIndexes.size());
 					InlineBlockChunkIndexes.emplace_back(std::vector<size_t>{ChunkIndex});
 				}
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 5fca3046c..3485b87cf 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -714,6 +714,7 @@ public:
 		if (Ctx.Settings.CollectSmallObjects)
 		{
 			BlockStore::BlockUsageMap BlockUsage;
+			BlockUsage.reserve(m_CasContainerStrategy.m_BlockStore.GetBlockCount());
 			{
 				RwLock::SharedLockScope __(m_CasContainerStrategy.m_LocationMapLock);
 				if (Ctx.IsCancelledFlag.load())
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index b0a4edf15..1276b9b4c 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -2413,7 +2413,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 														  Settings.AttachmentRangeMax));
 								}
 								SB.Append(fmt::format("      Cache attachment meta:     {}\n", Settings.StoreCacheAttachmentMetaData));
-								SB.Append(fmt::format("      Porject attachment meta:   {}\n", Settings.StoreProjectAttachmentMetaData));
+								SB.Append(fmt::format("      Project attachment meta:   {}\n", Settings.StoreProjectAttachmentMetaData));
 							}
 							SB.Append(fmt::format("      Cache cutoff time:         {}\n", Settings.CacheExpireTime));
 							SB.Append(fmt::format("      Project store cutoff time: {}", Settings.ProjectStoreExpireTime));
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index 5af9aeb10..40cfa55ce 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -96,10 +96,10 @@ struct BlockStoreFile : public RefCounted
 	void						 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Flush();
 	BasicFile&					 GetBasicFile();
-	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
-	bool	 IsOpen() const;
-	bool	 SetMetaData(const IoBuffer& Payload);
-	IoBuffer GetMetaData() const;
+	void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+	bool IsOpen() const;
+	bool SetMetaData(const IoBuffer& Payload);
+	bool GetMetaData(BasicFile& OutData) const;
 
 private:
 	std::filesystem::path		GetMetaPath() const;
@@ -154,6 +154,12 @@ public:
 
 	void Close();
 
+	size_t GetBlockCount() const
+	{
+		RwLock::SharedLockScope InsertLock(m_InsertLock);
+		return m_ChunkBlocks.size();
+	}
+
 	void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback);
 
 	typedef std::function<void(std::span<BlockStoreLocation> Locations)> WriteChunksCallback;
@@ -191,8 +197,8 @@ public:
 
 	bool IsWriting(uint32_t BlockIndex) const;
 
-	void	 SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload);
-	IoBuffer GetMetaData(uint32_t BlockIndex) const;
+	void SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload);
+	bool GetMetaData(uint32_t BlockIndex, BasicFile& OutData) const;
 
 private:
 	static const char*			 GetBlockFileExtension();
diff --git a/src/zenutil/include/zenutil/referencemetadata.h b/src/zenutil/include/zenutil/referencemetadata.h
index 5160bfb8b..986219571 100644
--- a/src/zenutil/include/zenutil/referencemetadata.h
+++ b/src/zenutil/include/zenutil/referencemetadata.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <zencore/compositebuffer.h>
+#include <zenutil/basicfile.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <gsl/gsl-lite.hpp>
@@ -44,7 +45,7 @@ struct ReferenceMetaDataHeader
 			   sizeof(AttachmentType) * AttachmentCount;
 	}
 
-	ReferenceMetaDataHeader() = delete;
+	ReferenceMetaDataHeader() = default;
 
 private:
 	uint32_t ComputeChecksum() const;
@@ -73,37 +74,101 @@ BuildReferenceMetaData(uint32_t						   HeaderMagic,
 
 template<typename KeyType, typename AttachmentType>
 bool
-GetAttachmentsFromMetaData(const IoBuffer&													  MetaData,
-						   const uint32_t													  ExpectedHeaderMagic,
+GetAttachmentsFromMetaData(BasicFile&																 MetaData,
+						   const uint32_t															 ExpectedHeaderMagic,
+						   std::function<void(std::uint32_t EntryCount, uint64_t AttachmentCount)>&& Prepare,
 						   std::function<void(std::span<const KeyType>		  Keys,
 											  std::span<const uint32_t>		  AttachmentCounts,
-											  std::span<const AttachmentType> Attachments)>&& Parser)
+											  std::span<const AttachmentType> Attachments)>&&		 Parser)
 {
-	MemoryView PayloadView = MetaData.GetView();
-	if (PayloadView.GetSize() >= sizeof(ReferenceMetaDataHeader))
+	uint64_t MetaDataSize = MetaData.FileSize();
+	if (MetaDataSize >= sizeof(ReferenceMetaDataHeader))
 	{
-		const ReferenceMetaDataHeader* Header = (const ReferenceMetaDataHeader*)PayloadView.GetData();
-		if (Header->IsValid(ExpectedHeaderMagic))
+		ReferenceMetaDataHeader Header;
+		uint64_t				Offset = 0;
+		MetaData.Read(&Header, sizeof(Header), Offset);
+		if (Header.IsValid(ExpectedHeaderMagic))
 		{
-			if (MetaData.GetSize() ==
-				ReferenceMetaDataHeader::ExpectedSize<KeyType, AttachmentType>(Header->EntryCount, Header->AttachmentCount))
+			Offset += sizeof(Header);
+			if (MetaDataSize == ReferenceMetaDataHeader::ExpectedSize<KeyType, AttachmentType>(Header.EntryCount, Header.AttachmentCount))
 			{
-				PayloadView.MidInline(sizeof(ReferenceMetaDataHeader));
-				std::span<const KeyType> PayloadKeys = {(const KeyType*)PayloadView.GetData(), Header->EntryCount};
-				PayloadView.MidInline(sizeof(KeyType) * Header->EntryCount);
-				std::span<const uint32_t> PayloadAttachmentCounts = {(const uint32_t*)PayloadView.GetData(), Header->EntryCount};
-				PayloadView.MidInline(sizeof(uint32_t) * Header->EntryCount);
-				std::span<const AttachmentType> PayloadAttachments = {(const AttachmentType*)PayloadView.GetData(),
-																	  Header->AttachmentCount};
-				PayloadView.MidInline(sizeof(AttachmentType) * Header->AttachmentCount);
-				ZEN_ASSERT(PayloadView.GetSize() == 0);
-
-				Parser(PayloadKeys, PayloadAttachmentCounts, PayloadAttachments);
-
+				if (Header.EntryCount > 0)
+				{
+					Prepare(Header.EntryCount, Header.AttachmentCount);
+
+					const uint64_t			   KeysSize = sizeof(KeyType) * Header.EntryCount;
+					std::unique_ptr<KeyType[]> Keys(std::make_unique<KeyType[]>(Header.EntryCount));
+					MetaData.Read(Keys.get(), KeysSize, Offset);
+					Offset += KeysSize;
+
+					const uint64_t				AttachmentCountsSize = sizeof(uint32_t) * Header.EntryCount;
+					std::unique_ptr<uint32_t[]> AttachmentCounts(std::make_unique<uint32_t[]>(Header.EntryCount));
+					MetaData.Read(AttachmentCounts.get(), AttachmentCountsSize, Offset);
+					Offset += AttachmentCountsSize;
+
+					const uint64_t					  MaxAttachmentsPerCall = (64u * 1024u) / sizeof(AttachmentType);
+					std::unique_ptr<AttachmentType[]> AttachmentRange(
+						std::make_unique<AttachmentType[]>(Min(MaxAttachmentsPerCall, Header.AttachmentCount)));
+
+					uint32_t KeyCount  = 0;
+					uint32_t KeyOffset = 0;
+					while (KeyOffset < Header.EntryCount)
+					{
+						uint64_t AttachmentRangeCount = 0;
+						while ((KeyOffset + KeyCount) < Header.EntryCount)
+						{
+							const uint32_t AttachmentCount = AttachmentCounts[KeyOffset + KeyCount];
+							if (AttachmentRangeCount + AttachmentCount > MaxAttachmentsPerCall)
+							{
+								break;
+							}
+							AttachmentRangeCount += AttachmentCount;
+							KeyCount++;
+						}
+
+						if (KeyCount == 0)
+						{
+							AttachmentRangeCount = AttachmentCounts[KeyOffset];
+							ZEN_ASSERT(AttachmentRangeCount > MaxAttachmentsPerCall);
+
+							std::span<const KeyType>  KeyRange(&Keys[KeyOffset], 1);
+							std::span<const uint32_t> AttachmentCountRange(&AttachmentCounts[KeyOffset], 1);
+
+							const uint64_t AttachmentRangeSize = sizeof(AttachmentType) * AttachmentRangeCount;
+							ZEN_ASSERT(Offset + AttachmentRangeSize <= MetaDataSize);
+							std::unique_ptr<AttachmentType[]> SingleAttachmentRange(
+								std::make_unique<AttachmentType[]>(AttachmentRangeCount));
+							MetaData.Read(SingleAttachmentRange.get(), AttachmentRangeSize, Offset);
+							Offset += AttachmentRangeSize;
+
+							std::span<const AttachmentType> AttachmentsRange(SingleAttachmentRange.get(), AttachmentRangeCount);
+
+							Parser(KeyRange, AttachmentCountRange, AttachmentsRange);
+							KeyOffset++;
+						}
+						else if (AttachmentRangeCount > 0)
+						{
+							std::span<const KeyType>  KeyRange(&Keys[KeyOffset], KeyCount);
+							std::span<const uint32_t> AttachmentCountRange(&AttachmentCounts[KeyOffset], KeyCount);
+
+							const uint64_t AttachmentRangeSize = sizeof(AttachmentType) * AttachmentRangeCount;
+							ZEN_ASSERT(Offset + AttachmentRangeSize <= MetaDataSize);
+							MetaData.Read(AttachmentRange.get(), AttachmentRangeSize, Offset);
+							Offset += AttachmentRangeSize;
+
+							std::span<const AttachmentType> AttachmentsRange(AttachmentRange.get(), AttachmentRangeCount);
+
+							Parser(KeyRange, AttachmentCountRange, AttachmentsRange);
+							KeyOffset += KeyCount;
+						}
+						KeyCount = 0;
+					}
+				}
 				return true;
 			}
 		}
 	}
 	return false;
 }
+
 }  // namespace zen