flush file cas on exit (#291)

* flush caslog and index snapshot on flush
* fix save reading of index/logfile with cleanup
write snapshot at flush
* don't validate entries we just scanned/created
* fix total size found when scanning for cas files
* changelog

2 files changed, 33 insertions, 24 deletions

diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 1d25920c4..108d37607 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -179,8 +179,31 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 		}
 	}
 
-	m_LogFlushPosition	   = ReadIndexFile();
-	uint64_t LogEntryCount = ReadLog(m_LogFlushPosition);
+	if (std::filesystem::is_regular_file(IndexPath))
+	{
+		uint32_t IndexVersion = 0;
+		m_LogFlushPosition	  = ReadIndexFile(IndexPath, IndexVersion);
+		if (IndexVersion == 0)
+		{
+			ZEN_WARN("removing invalid index file at '{}'", IndexPath);
+			std::filesystem::remove(IndexPath);
+		}
+	}
+
+	uint64_t LogEntryCount = 0;
+	if (std::filesystem::is_regular_file(LogPath))
+	{
+		if (TCasLogFile<FileCasIndexEntry>::IsValid(LogPath))
+		{
+			LogEntryCount = ReadLog(LogPath, m_LogFlushPosition);
+		}
+		else
+		{
+			ZEN_WARN("removing invalid cas log at '{}'", LogPath);
+			std::filesystem::remove(LogPath);
+		}
+	}
+
 	for (const auto& Entry : m_Index)
 	{
 		m_TotalSize.fetch_add(Entry.second.Size, std::memory_order::relaxed);
@@ -804,16 +827,8 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, IoBuffer&&
 void
 FileCasStrategy::Flush()
 {
-	// Since we don't keep files open after writing there's nothing specific
-	// to flush here.
-	//
-	// Depending on what semantics we want Flush() to provide, it could be
-	// argued that this should just flush the volume which we are using to
-	// store the CAS files on here, to ensure metadata is flushed along
-	// with file data
-	//
-	// Related: to facilitate more targeted validation during recovery we could
-	// maintain a log of when chunks were created
+	m_CasLog.Flush();
+	MakeIndexSnapshot();
 }
 
 void
@@ -1095,12 +1110,11 @@ FileCasStrategy::MakeIndexSnapshot()
 	}
 }
 uint64_t
-FileCasStrategy::ReadIndexFile()
+FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion)
 {
 	using namespace filecas::impl;
 
 	std::vector<FileCasIndexEntry> Entries;
-	std::filesystem::path		   IndexPath = GetIndexPath(m_RootDirectory);
 	if (std::filesystem::is_regular_file(IndexPath))
 	{
 		Stopwatch  Timer;
@@ -1135,7 +1149,7 @@ FileCasStrategy::ReadIndexFile()
 					}
 					m_Index.insert_or_assign(Entry.Key, IndexEntry{.Size = Entry.Size});
 				}
-
+				OutVersion = FileCasIndexHeader::CurrentVersion;
 				return Header.LogPosition;
 			}
 			else
@@ -1167,12 +1181,8 @@ FileCasStrategy::ReadIndexFile()
 		std::string InvalidEntryReason;
 		for (const FileCasStrategy::FileCasIndexEntry& Entry : ScannedEntries)
 		{
-			if (!ValidateEntry(Entry, InvalidEntryReason))
-			{
-				ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", m_RootDirectory, InvalidEntryReason);
-				continue;
-			}
 			m_Index.insert_or_assign(Entry.Key, IndexEntry{.Size = Entry.Size});
+			TotalSize += Entry.Size;
 			CasLog.Append(Entry);
 		}
 
@@ -1183,11 +1193,10 @@ FileCasStrategy::ReadIndexFile()
 }
 
 uint64_t
-FileCasStrategy::ReadLog(uint64_t SkipEntryCount)
+FileCasStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntryCount)
 {
 	using namespace filecas::impl;
 
-	std::filesystem::path LogPath = GetLogPath(m_RootDirectory);
 	if (std::filesystem::is_regular_file(LogPath))
 	{
 		uint64_t					   LogEntryCount = 0;
diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h
index 420b3a634..01d3648da 100644
--- a/src/zenstore/filecas.h
+++ b/src/zenstore/filecas.h
@@ -46,8 +46,8 @@ struct FileCasStrategy final : public GcStorage
 
 private:
 	void	 MakeIndexSnapshot();
-	uint64_t ReadIndexFile();
-	uint64_t ReadLog(uint64_t LogPosition);
+	uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
+	uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
 
 	struct IndexEntry
 	{