projectstore: added validation of oplog data during initialization

also added validation during writing

5 files changed, 93 insertions, 10 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index b120f3955..1b6c30cbd 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -529,15 +529,28 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
 	m_SlogFile.Open(SlogPath, IsNew);
 
 	uint64_t MaxFileOffset = 0;
+	uint64_t InvalidEntryCount = 0;
 
 	if (RwLock::ExclusiveLockScope _(m_IndexLock); m_Index.empty())
 	{
 		m_SlogFile.Replay([&](const DiskIndexEntry& Record) {
-			m_Index[Record.Key] = Record.Location;
+			if (Record.Key == IoHash::Zero)
+			{
+				++InvalidEntryCount;
+			}
+			else
+			{
+				m_Index[Record.Key] = Record.Location;
 
-			MaxFileOffset = std::max<uint64_t>(MaxFileOffset, Record.Location.Offset() + Record.Location.Size());
+				MaxFileOffset = std::max<uint64_t>(MaxFileOffset, Record.Location.Offset() + Record.Location.Size());
+			}
 		});
 
+		if (InvalidEntryCount)
+		{
+			ZEN_WARN("found {} invalid entries in '{}'", InvalidEntryCount, SlogPath);
+		}
+
 		m_WriteCursor = (MaxFileOffset + 15) & ~15;
 	}
 
@@ -998,7 +1011,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 	{
 		// New bucket needs to be created
 
-		std::string BucketName8 = WideToUtf8(BucketName);
+ 		const std::string BucketName8 = ToUtf8(BucketName);
 
 		if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end())
 		{
@@ -1014,7 +1027,11 @@ ZenCacheDiskLayer::DiscoverBuckets()
 
 			Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false);
 
-			if (!Bucket.IsOk())
+			if (Bucket.IsOk())
+			{
+				ZEN_INFO("Discovered bucket '{}'", BucketName8);
+			}
+			else
 			{
 				ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName8, m_RootDir);
 
diff --git a/zenserver/config.h b/zenserver/config.h
index 72a4f31bb..7fa8163b3 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -9,6 +9,14 @@
 #	define ZEN_ENABLE_MESH 0
 #endif
 
+#ifndef ZEN_USE_NAMED_PIPES
+#	define ZEN_USE_NAMED_PIPES 0
+#endif
+
+#ifndef ZEN_USE_EXEC
+#	define ZEN_USE_EXEC 0
+#endif
+
 struct ZenServerOptions
 {
 	bool				  IsDebug	  = false;
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 72d7284a0..3824da63a 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -10,14 +10,20 @@
 #include <zencore/logging.h>
 #include <zencore/stream.h>
 #include <zencore/string.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
 #include <zencore/timer.h>
 #include <zencore/windows.h>
 #include <zenstore/basicfile.h>
 #include <zenstore/cas.h>
 #include <zenstore/caslog.h>
 
+#include "config.h"
+
 #define USE_ROCKSDB 0
 
+ZEN_THIRD_PARTY_INCLUDES_START
+
 #if USE_ROCKSDB
 #	pragma comment(lib, "Rpcrt4.lib")	// RocksDB made me do this
 #	include <rocksdb/db.h>
@@ -25,6 +31,8 @@
 
 #include <xxh3.h>
 #include <asio.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 #include <latch>
 #include <string>
 
@@ -165,7 +173,16 @@ struct ProjectStore::OplogStorage : public RefCounted
 
 		Stopwatch Timer;
 
+		uint64_t InvalidEntries = 0;
+
 		m_Oplog.Replay([&](const zen::OplogEntry& LogEntry) {
+			if (LogEntry.OpCoreSize == 0)
+			{
+				++InvalidEntries;
+
+				return;
+			}
+
 			IoBuffer OpBuffer(LogEntry.OpCoreSize);
 
 			const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign;
@@ -190,6 +207,11 @@ struct ProjectStore::OplogStorage : public RefCounted
 			Handler(Op, LogEntry);
 		});
 
+		if (InvalidEntries)
+		{
+			ZEN_WARN("ignored {} zero-sized oplog entries", InvalidEntries);
+		}
+
 		ZEN_INFO("Oplog replay completed in {} - Max LSN# {}, Next offset: {}",
 				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 				 m_MaxLsn,
@@ -221,6 +243,8 @@ struct ProjectStore::OplogStorage : public RefCounted
 		const uint64_t WriteSize  = Buffer.GetSize();
 		const auto	   OpCoreHash = uint32_t(XXH3_64bits(Buffer.GetData(), WriteSize) & 0xffffFFFF);
 
+		ZEN_ASSERT(WriteSize != 0);
+
 		XXH3_128Stream KeyHasher;
 		Op["key"].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
 		XXH3_128 KeyHash = KeyHasher.GetHash();
@@ -305,7 +329,7 @@ ProjectStore::Oplog::Scrub(ScrubContext& Ctx) const
 }
 
 void
-ProjectStore::Oplog::GatherReferences(GcContext& GcCtx) const
+ProjectStore::Oplog::GatherReferences(GcContext& GcCtx)
 {
 	RwLock::SharedLockScope _(m_OplogLock);
 
@@ -791,6 +815,19 @@ ProjectStore::Project::IterateOplogs(std::function<void(const Oplog&)>&& Fn) con
 }
 
 void
+ProjectStore::Project::IterateOplogs(std::function<void(Oplog&)>&& Fn)
+{
+	// TODO: should also iterate over oplogs which are present on disk but not yet loaded
+
+	RwLock::SharedLockScope _(m_ProjectLock);
+
+	for (auto& Kv : m_Oplogs)
+	{
+		Fn(Kv.second);
+	}
+}
+
+void
 ProjectStore::Project::Flush()
 {
 	// TODO
@@ -805,7 +842,7 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx)
 void
 ProjectStore::Project::GatherReferences(GcContext& GcCtx)
 {
-	IterateOplogs([&](const Oplog& Ops) { Ops.GatherReferences(GcCtx); });
+	IterateOplogs([&](Oplog& Ops) { Ops.GatherReferences(GcCtx); });
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -1726,6 +1763,8 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
 	}
 }
 
+#if ZEN_USE_NAMED_PIPES
+
 //////////////////////////////////////////////////////////////////////////
 
 class SecurityAttributes
@@ -2005,6 +2044,25 @@ LocalProjectService::~LocalProjectService()
 	m_Impl->Stop();
 }
 
+#endif
+
 //////////////////////////////////////////////////////////////////////////
 
+#if ZEN_WITH_TESTS
+
+TEST_CASE("prj.store")
+{
+	using namespace fmt::literals;
+	using namespace std::literals;
+
+	ScopedTemporaryDirectory TempDir;
+}
+
+#endif
+
+void
+prj_forcelink()
+{
+}
+
 }  // namespace zen
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index 743abb5eb..6bd210573 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -107,7 +107,7 @@ public:
 		spdlog::logger& Log() { return m_OuterProject->Log(); }
 		void			Flush();
 		void			Scrub(ScrubContext& Ctx) const;
-		void			GatherReferences(GcContext& GcCtx) const;
+		void			GatherReferences(GcContext& GcCtx);
 
 		std::size_t OplogCount() const { return m_LatestOpMap.size(); }
 
@@ -153,6 +153,7 @@ public:
 		Oplog* OpenOplog(std::string_view OplogId);
 		void   DeleteOplog(std::string_view OplogId);
 		void   IterateOplogs(std::function<void(const Oplog&)>&& Fn) const;
+		void   IterateOplogs(std::function<void(Oplog&)>&& Fn);
 
 		Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath);
 		~Project();
@@ -265,4 +266,6 @@ private:
 	std::unique_ptr<LocalProjectImpl> m_Impl;
 };
 
+void prj_forcelink();
+
 }  // namespace zen
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 8da781a59..92d233f5b 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -19,9 +19,6 @@
 #include <zenstore/cidstore.h>
 #include <zenutil/zenserverprocess.h>
 
-#define ZEN_USE_NAMED_PIPES 0
-#define ZEN_USE_EXEC		0
-
 #if ZEN_USE_MIMALLOC
 ZEN_THIRD_PARTY_INCLUDES_START
 #	include <mimalloc-new-delete.h>