Fix log index snapshot (#210)

* Fix log reading for structured cache store
Make sure cache is flushed at exit
* dont flush index to disk unless new entries have been written
* changelog

1 files changed, 17 insertions, 14 deletions

diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 1c692b609..70a88ecad 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -228,6 +228,7 @@ void
 CasContainerStrategy::Flush()
 {
 	m_BlockStore.Flush();
+	m_CasLog.Flush();
 	MakeIndexSnapshot();
 }
 
@@ -502,6 +503,12 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 void
 CasContainerStrategy::MakeIndexSnapshot()
 {
+	uint64_t LogCount = m_CasLog.GetLogCount();
+	if (m_LogFlushPosition == LogCount)
+	{
+		return;
+	}
+
 	ZEN_DEBUG("write store snapshot for '{}'", m_RootDirectory / m_ContainerBaseName);
 	uint64_t   EntryCount = 0;
 	Stopwatch  Timer;
@@ -529,10 +536,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 
 	try
 	{
-		m_CasLog.Flush();
-
 		// Write the current state of the location map to a new index state
-		uint64_t					   LogCount = 0;
 		std::vector<CasDiskIndexEntry> Entries;
 
 		{
@@ -546,8 +550,6 @@ CasContainerStrategy::MakeIndexSnapshot()
 				IndexEntry.Key				  = Entry.first;
 				IndexEntry.Location			  = Entry.second;
 			}
-
-			LogCount = m_CasLog.GetLogCount();
 		}
 
 		BasicFile ObjectIndexFile;
@@ -562,7 +564,8 @@ CasContainerStrategy::MakeIndexSnapshot()
 		ObjectIndexFile.Write(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexEntry));
 		ObjectIndexFile.Flush();
 		ObjectIndexFile.Close();
-		EntryCount = Entries.size();
+		EntryCount		   = Entries.size();
+		m_LogFlushPosition = LogCount;
 	}
 	catch (std::exception& Err)
 	{
@@ -638,15 +641,15 @@ CasContainerStrategy::ReadIndexFile()
 uint64_t
 CasContainerStrategy::ReadLog(uint64_t SkipEntryCount)
 {
-	std::vector<CasDiskIndexEntry> Entries;
-	std::filesystem::path		   LogPath = GetLogPath(m_RootDirectory, m_ContainerBaseName);
+	std::filesystem::path LogPath = GetLogPath(m_RootDirectory, m_ContainerBaseName);
 	if (std::filesystem::is_regular_file(LogPath))
 	{
+		size_t	   LogEntryCount = 0;
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			ZEN_INFO("read store '{}' log containing #{} entries in {}",
 					 m_RootDirectory / m_ContainerBaseName,
-					 Entries.size(),
+					 LogEntryCount,
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
@@ -660,10 +663,10 @@ CasContainerStrategy::ReadLog(uint64_t SkipEntryCount)
 				ZEN_WARN("reading full log at '{}', reason: Log position from index snapshot is out of range", LogPath);
 				SkipEntryCount = 0;
 			}
-			uint64_t ReadCount = EntryCount - SkipEntryCount;
-			Entries.reserve(ReadCount);
+			LogEntryCount = EntryCount - SkipEntryCount;
 			CasLog.Replay(
 				[&](const CasDiskIndexEntry& Record) {
+					LogEntryCount++;
 					std::string InvalidEntryReason;
 					if (Record.Flags & CasDiskIndexEntry::kTombstone)
 					{
@@ -678,7 +681,7 @@ CasContainerStrategy::ReadLog(uint64_t SkipEntryCount)
 					m_LocationMap[Record.Key] = Record.Location;
 				},
 				SkipEntryCount);
-			return ReadCount;
+			return LogEntryCount;
 		}
 	}
 	return 0;
@@ -698,8 +701,8 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 		std::filesystem::remove_all(BasePath);
 	}
 
-	uint64_t LogPosition   = ReadIndexFile();
-	uint64_t LogEntryCount = ReadLog(LogPosition);
+	m_LogFlushPosition	   = ReadIndexFile();
+	uint64_t LogEntryCount = ReadLog(m_LogFlushPosition);
 
 	CreateDirectories(BasePath);