store cache rawhash and rawsize for unstructured cache values (#234)

* refactored MemoryCacheBucket to allow for storing RawHash/RawSize.
* remove redundant conversions in AccessTime
* reduce max count for memory cache bucket to 32-bit value
* refactored DiskCacheBucket to allow for storing RawHash/RawSize.
* Use CompressedBuffer::ValidateCompressedHeader when applicable
* Make sure we rewrite the snapshot if we read an legacy existing index/log
* changelog

2 files changed, 53 insertions, 9 deletions

diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp
index 9c5258bce..2a978ae12 100644
--- a/zenstore/caslog.cpp
+++ b/zenstore/caslog.cpp
@@ -38,6 +38,42 @@ CasLogFile::~CasLogFile()
 {
 }
 
+bool
+CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize)
+{
+	if (!std::filesystem::is_regular_file(FileName))
+	{
+		return false;
+	}
+	BasicFile File;
+
+	std::error_code Ec;
+	File.Open(FileName, BasicFile::Mode::kRead, Ec);
+	if (Ec)
+	{
+		return false;
+	}
+
+	FileHeader Header;
+	if (File.FileSize() < sizeof(Header))
+	{
+		return false;
+	}
+
+	// Validate header and log contents and prepare for appending/replay
+	File.Read(&Header, sizeof Header, 0);
+
+	if ((0 != memcmp(Header.Magic, FileHeader::MagicSequence, sizeof Header.Magic)) || (Header.Checksum != Header.ComputeChecksum()))
+	{
+		return false;
+	}
+	if (Header.RecordSize != RecordSize)
+	{
+		return false;
+	}
+	return true;
+}
+
 void
 CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode)
 {
@@ -82,7 +118,6 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode)
 	}
 	else
 	{
-		// Validate header and log contents and prepare for appending/replay
 		FileHeader Header;
 		m_File.Read(&Header, sizeof Header, 0);
 
@@ -90,6 +125,13 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode)
 		{
 			throw std::runtime_error(fmt::format("Mangled log header (invalid header magic) in '{}'", FileName));
 		}
+		if (Header.RecordSize != RecordSize)
+		{
+			throw std::runtime_error(fmt::format("Mangled log header (mismatch in record size, expected {}, found {}) in '{}'",
+												 RecordSize,
+												 Header.RecordSize,
+												 FileName));
+		}
 
 		AppendOffset = m_File.FileSize();
 
diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h
index 1aeb50d05..d8c3f22f3 100644
--- a/zenstore/include/zenstore/caslog.h
+++ b/zenstore/include/zenstore/caslog.h
@@ -20,13 +20,14 @@ public:
 		kTruncate
 	};
 
-	void	 Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode);
-	void	 Append(const void* DataPointer, uint64_t DataSize);
-	void	 Replay(std::function<void(const void*)>&& Handler, uint64_t SkipEntryCount);
-	void	 Flush();
-	void	 Close();
-	uint64_t GetLogSize();
-	uint64_t GetLogCount();
+	static bool IsValid(std::filesystem::path FileName, size_t RecordSize);
+	void		Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode);
+	void		Append(const void* DataPointer, uint64_t DataSize);
+	void		Replay(std::function<void(const void*)>&& Handler, uint64_t SkipEntryCount);
+	void		Flush();
+	void		Close();
+	uint64_t	GetLogSize();
+	uint64_t	GetLogCount();
 
 private:
 	struct FileHeader
@@ -59,7 +60,8 @@ template<typename T>
 class TCasLogFile : public CasLogFile
 {
 public:
-	void Open(std::filesystem::path FileName, Mode Mode) { CasLogFile::Open(FileName, sizeof(T), Mode); }
+	static bool IsValid(std::filesystem::path FileName) { return CasLogFile::IsValid(FileName, sizeof(T)); }
+	void		Open(std::filesystem::path FileName, Mode Mode) { CasLogFile::Open(FileName, sizeof(T), Mode); }
 
 	// This should be called before the Replay() is called to do some basic sanity checking
 	bool Initialize() { return true; }