keep all block files open from start

1 files changed, 14 insertions, 7 deletions

diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 2bf96ce9d..f7cfed47b 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -623,15 +623,22 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const
 			m_CurrentInsertOffset = 0;
 			m_WriteBlock->Create(m_MaxBlockSize);
 		}
-		else
-		{
-			m_WriteBlock->Open();
-		}
 		InsertOffset		  = m_CurrentInsertOffset;
 		m_CurrentInsertOffset = RoundUp(InsertOffset + ChunkSize, m_PayloadAlignment);
 		WriteBlock			  = m_WriteBlock;
 	}
 
+	// We can end up in a situation that two InsertChunk writes the same chunk
+	// data in different locations.
+	// We release the insert lock once we have the correct WriteBlock ready and we know
+	// where to write the data. If a new InsertChunk requests comes in before we update
+	// m_LocationMap below we will have a race.
+	// The outcome of that is that we will occupy space more than once but the hash will
+	// only point to one of the chunks. We will in that case waste space until the next
+	// GC operation.
+	// This would be a rare occasion and the current flow reduces the time we block for
+	// reads, insert and GC.
+
 	BlockStoreDiskLocation	Location({.BlockIndex = WriteBlockIndex, .Offset = InsertOffset, .Size = ChunkSize}, m_PayloadAlignment);
 	const CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = Location};
 
@@ -1052,8 +1059,6 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 			continue;
 		}
 
-		OldBlockFile->Open();
-
 		std::vector<uint8_t> Chunk;
 		for (const IoHash& ChunkHash : KeepMap)
 		{
@@ -1067,6 +1072,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 				uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order::memory_order_relaxed);
 				auto	 LogEntries		= MakeCasDiskEntries(MovedBlockChunks, {});
 				m_CasLog.Append(LogEntries);
+
 				{
 					RwLock::ExclusiveLockScope __(m_LocationMapLock);
 					Stopwatch				   Timer;
@@ -1098,7 +1104,6 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 					ZEN_ERROR("get disk space in {} FAILED, reason '{}'", m_ContainerBaseName, Error.message());
 					return;
 				}
-
 				if (Space.Free < m_MaxBlockSize)
 				{
 					std::filesystem::path GCReservePath = GetGCReservePath(m_Config.RootDirectory, m_ContainerBaseName);
@@ -1130,6 +1135,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 				WriteOffset	  = 0;
 			}
 
+
 			NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
 			MovedBlockChunks.emplace(
 				ChunkHash,
@@ -1435,6 +1441,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 					}
 					auto BlockPath			  = GetBlockPath(m_BlocksBasePath, BlockIndex);
 					auto BlockFile			  = std::make_shared<BlockStoreFile>(BlockPath);
+					BlockFile->Open();
 					m_ChunkBlocks[BlockIndex] = BlockFile;
 				}
 			}