missing chunks bugfix (#424)

* make sure to close log file when resetting log
* drop entries that refers to missing blocks
* Don't scrub keys that has been rewritten
* currectly count added bytes / m_TotalSize
* fix negative sleep time in BlockStoreFile::Open()
* be defensive when fetching log position
* append to log files *after* we updated all state successfully
* explicitly close stuff in destructors with exception catching
* clean up empty size block store files

1 files changed, 37 insertions, 16 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 5081ae65d..7b56c64bd 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -70,7 +70,7 @@ BlockStoreFile::Open()
 			return false;
 		}
 		ZEN_WARN("Failed to open cas block '{}', reason: '{}', retries left: {}.", m_Path, Ec.message(), RetriesLeft);
-		Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
+		Sleep(100 + (3 - RetriesLeft) * 100);  // Total 600 ms
 		RetriesLeft--;
 		return true;
 	});
@@ -286,6 +286,14 @@ BlockStore::BlockStore()
 
 BlockStore::~BlockStore()
 {
+	try
+	{
+		Close();
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("~BlockStore() failed with: ", Ex.what());
+	}
 }
 
 void
@@ -307,6 +315,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 
 	if (IsDir(m_BlocksBasePath))
 	{
+		std::vector<std::filesystem::path> EmptyBlockFiles;
 		uint32_t						   NextBlockIndex = 0;
 		std::vector<std::filesystem::path> FoldersToScan;
 		FoldersToScan.push_back(m_BlocksBasePath);
@@ -334,6 +343,12 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 					{
 						continue;
 					}
+					if (Entry.file_size() == 0)
+					{
+						EmptyBlockFiles.push_back(Path);
+						continue;
+					}
+
 					Ref<BlockStoreFile> BlockFile{new BlockStoreFile(Path)};
 					BlockFile->Open();
 					m_TotalSize.fetch_add(BlockFile->TotalSize(), std::memory_order::relaxed);
@@ -347,6 +362,17 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 			}
 			++FolderOffset;
 		}
+
+		for (const std::filesystem::path& EmptyBlockFile : EmptyBlockFiles)
+		{
+			std::error_code Ec;
+			RemoveFile(EmptyBlockFile, Ec);
+			if (Ec)
+			{
+				ZEN_WARN("Unable to remove empty block file {}. Reason: {}", EmptyBlockFile, Ec.message());
+			}
+		}
+
 		m_WriteBlockIndex.store(NextBlockIndex, std::memory_order_release);
 	}
 	else
@@ -355,7 +381,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 	}
 }
 
-void
+BlockStore::BlockIndexSet
 BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks)
 {
 	ZEN_MEMSCOPE(GetBlocksTag());
@@ -363,8 +389,8 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks)
 
 	RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
 
-	tsl::robin_set<uint32_t> MissingBlocks;
-	tsl::robin_set<uint32_t> DeleteBlocks;
+	BlockIndexSet MissingBlocks;
+	BlockIndexSet DeleteBlocks;
 	DeleteBlocks.reserve(m_ChunkBlocks.size());
 	for (auto It : m_ChunkBlocks)
 	{
@@ -383,13 +409,6 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks)
 			MissingBlocks.insert(BlockIndex);
 		}
 	}
-	for (std::uint32_t BlockIndex : MissingBlocks)
-	{
-		std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex);
-		Ref<BlockStoreFile>	  NewBlockFile(new BlockStoreFile(BlockPath));
-		NewBlockFile->Create(0);
-		m_ChunkBlocks[BlockIndex] = NewBlockFile;
-	}
 	for (std::uint32_t BlockIndex : DeleteBlocks)
 	{
 		std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex);
@@ -400,6 +419,7 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks)
 		}
 		m_ChunkBlocks.erase(BlockIndex);
 	}
+	return MissingBlocks;
 }
 
 BlockStore::BlockEntryCountMap
@@ -1037,6 +1057,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 		{
 			Continue = ChangeCallback(MovedChunks, ScrubbedChunks, RemovedSize > AddedSize ? RemovedSize - AddedSize : 0);
 			DeletedSize += RemovedSize;
+			m_TotalSize.fetch_add(AddedSize);
 			RemovedSize = 0;
 			AddedSize	= 0;
 			MovedCount += MovedChunks.size();
@@ -1220,14 +1241,13 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 								 NiceBytes(Space.Free + ReclaimedSpace));
 					}
 					NewBlockFile->Create(m_MaxBlockSize);
-					NewBlockIndex = NextBlockIndex;
-					WriteOffset	  = 0;
-					AddedSize += WriteOffset;
+					NewBlockIndex	 = NextBlockIndex;
 					WriteOffset		 = 0;
 					TargetFileBuffer = std::make_unique<BasicFileWriter>(NewBlockFile->GetBasicFile(), Min(256u * 1024u, m_MaxBlockSize));
 				}
 
-				WriteOffset = TargetFileBuffer->AlignTo(PayloadAlignment);
+				const uint64_t OldWriteOffset = WriteOffset;
+				WriteOffset					  = TargetFileBuffer->AlignTo(PayloadAlignment);
 
 				TargetFileBuffer->Write(ChunkView.GetData(), ChunkLocation.Size, WriteOffset);
 				MovedChunks.push_back(
@@ -1235,8 +1255,9 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 				WriteOffset += ChunkLocation.Size;
 
 				MovedFromBlock += RoundUp(ChunkLocation.Offset + ChunkLocation.Size, PayloadAlignment) - ChunkLocation.Offset;
+				uint64_t WrittenBytes = WriteOffset - OldWriteOffset;
+				AddedSize += WrittenBytes;
 			}
-			AddedSize += WriteOffset;
 			ZEN_INFO("{}moved {} chunks ({}) from '{}' to new block, freeing {}",
 					 LogPrefix,
 					 KeepChunkIndexes.size(),