projectstore: added validation of oplog data during initialization

also added validation during writing

1 files changed, 60 insertions, 2 deletions

diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 72d7284a0..3824da63a 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -10,14 +10,20 @@
 #include <zencore/logging.h>
 #include <zencore/stream.h>
 #include <zencore/string.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
 #include <zencore/timer.h>
 #include <zencore/windows.h>
 #include <zenstore/basicfile.h>
 #include <zenstore/cas.h>
 #include <zenstore/caslog.h>
 
+#include "config.h"
+
 #define USE_ROCKSDB 0
 
+ZEN_THIRD_PARTY_INCLUDES_START
+
 #if USE_ROCKSDB
 #	pragma comment(lib, "Rpcrt4.lib")	// RocksDB made me do this
 #	include <rocksdb/db.h>
@@ -25,6 +31,8 @@
 
 #include <xxh3.h>
 #include <asio.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 #include <latch>
 #include <string>
 
@@ -165,7 +173,16 @@ struct ProjectStore::OplogStorage : public RefCounted
 
 		Stopwatch Timer;
 
+		uint64_t InvalidEntries = 0;
+
 		m_Oplog.Replay([&](const zen::OplogEntry& LogEntry) {
+			if (LogEntry.OpCoreSize == 0)
+			{
+				++InvalidEntries;
+
+				return;
+			}
+
 			IoBuffer OpBuffer(LogEntry.OpCoreSize);
 
 			const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign;
@@ -190,6 +207,11 @@ struct ProjectStore::OplogStorage : public RefCounted
 			Handler(Op, LogEntry);
 		});
 
+		if (InvalidEntries)
+		{
+			ZEN_WARN("ignored {} zero-sized oplog entries", InvalidEntries);
+		}
+
 		ZEN_INFO("Oplog replay completed in {} - Max LSN# {}, Next offset: {}",
 				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 				 m_MaxLsn,
@@ -221,6 +243,8 @@ struct ProjectStore::OplogStorage : public RefCounted
 		const uint64_t WriteSize  = Buffer.GetSize();
 		const auto	   OpCoreHash = uint32_t(XXH3_64bits(Buffer.GetData(), WriteSize) & 0xffffFFFF);
 
+		ZEN_ASSERT(WriteSize != 0);
+
 		XXH3_128Stream KeyHasher;
 		Op["key"].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
 		XXH3_128 KeyHash = KeyHasher.GetHash();
@@ -305,7 +329,7 @@ ProjectStore::Oplog::Scrub(ScrubContext& Ctx) const
 }
 
 void
-ProjectStore::Oplog::GatherReferences(GcContext& GcCtx) const
+ProjectStore::Oplog::GatherReferences(GcContext& GcCtx)
 {
 	RwLock::SharedLockScope _(m_OplogLock);
 
@@ -791,6 +815,19 @@ ProjectStore::Project::IterateOplogs(std::function<void(const Oplog&)>&& Fn) con
 }
 
 void
+ProjectStore::Project::IterateOplogs(std::function<void(Oplog&)>&& Fn)
+{
+	// TODO: should also iterate over oplogs which are present on disk but not yet loaded
+
+	RwLock::SharedLockScope _(m_ProjectLock);
+
+	for (auto& Kv : m_Oplogs)
+	{
+		Fn(Kv.second);
+	}
+}
+
+void
 ProjectStore::Project::Flush()
 {
 	// TODO
@@ -805,7 +842,7 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx)
 void
 ProjectStore::Project::GatherReferences(GcContext& GcCtx)
 {
-	IterateOplogs([&](const Oplog& Ops) { Ops.GatherReferences(GcCtx); });
+	IterateOplogs([&](Oplog& Ops) { Ops.GatherReferences(GcCtx); });
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -1726,6 +1763,8 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
 	}
 }
 
+#if ZEN_USE_NAMED_PIPES
+
 //////////////////////////////////////////////////////////////////////////
 
 class SecurityAttributes
@@ -2005,6 +2044,25 @@ LocalProjectService::~LocalProjectService()
 	m_Impl->Stop();
 }
 
+#endif
+
 //////////////////////////////////////////////////////////////////////////
 
+#if ZEN_WITH_TESTS
+
+TEST_CASE("prj.store")
+{
+	using namespace fmt::literals;
+	using namespace std::literals;
+
+	ScopedTemporaryDirectory TempDir;
+}
+
+#endif
+
+void
+prj_forcelink()
+{
+}
+
 }  // namespace zen