make oplog lsn unsigned (#107)

* change oplog lsn to uint32

4 files changed, 35 insertions, 45 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e54c6fd9c..ac20da127 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@
 - Improvement: Add hardening to gracefully handle malformed oplogs in project store
 - Improvement: Catch exceptions in threaded work to avoid uncaught exception errors
 - Improvement: Make oplog/project removal more robust
+- Improvement: Made LSN number for project store oplog an unsigned 32 bit value at the top layer to increase range
 
 ## 5.5.3
 - Feature: New 'workspaces' service which allows a user to share a local folder via zenserver. A workspace can have mulitple workspace shares and they provie an HTTP API that is compatible with the project oplog HTTP API. Workspaces and shares are preserved between runs. Workspaces feature is disabled by default - enable with `--workspaces-enabled` option when launching zenserver.
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 585cee26f..6de91cf94 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -56,7 +56,7 @@ CSVWriteOp(CidStore&		  CidStore,
 		   std::string_view	  OplogId,
 		   bool				  Details,
 		   bool				  AttachmentDetails,
-		   int				  LSN,
+		   uint32_t			  LSN,
 		   const Oid&		  Key,
 		   CbObjectView		  Op,
 		   StringBuilderBase& CSVWriter)
@@ -103,7 +103,7 @@ namespace {
 				   bool			   Details,
 				   bool			   OpDetails,
 				   bool			   AttachmentDetails,
-				   int			   LSN,
+				   uint32_t		   LSN,
 				   const Oid&	   Key,
 				   CbObjectView	   Op,
 				   CbObjectWriter& CbWriter)
@@ -175,9 +175,10 @@ namespace {
 	{
 		Cbo.BeginArray("ops");
 		{
-			Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](int LSN, const Oid& Key, CbObjectView Op) {
-				CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
-			});
+			Oplog.IterateOplogWithKey(
+				[&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+					CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
+				});
 		}
 		Cbo.EndArray();
 	}
@@ -1782,7 +1783,7 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
 		m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
 			Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
 				Oplog.IterateOplogWithKey(
-					[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObjectView Op) {
+					[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
 						CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
 					});
 			});
@@ -1837,7 +1838,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
 
 		FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
 			Oplog.IterateOplogWithKey(
-				[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObjectView Op) {
+				[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
 					CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
 				});
 		});
@@ -1893,7 +1894,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
 		Oplog.IterateOplogWithKey(
-			[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObjectView Op) {
+			[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
 				CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
 			});
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
@@ -1952,13 +1953,13 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 	ProjectStore::Project& Project = *FoundProject.Get();
 	ProjectStore::Oplog&   Oplog   = *FoundLog;
 
-	int LSN = Oplog.GetOpIndexByKey(ObjId);
-	if (LSN == -1)
+	std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId);
+	if (!Op.has_value())
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
-	std::optional<CbObject> Op = Oplog.GetOpByIndex(LSN);
-	if (!Op.has_value())
+	std::optional<uint32_t> LSN = Oplog.GetOpIndexByKey(ObjId);
+	if (!LSN.has_value())
 	{
 		return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 	}
@@ -1968,7 +1969,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		ExtendableStringBuilder<4096> CSVWriter;
 		CSVHeader(Details, AttachmentDetails, CSVWriter);
 
-		CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter);
+		CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
 		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
 	}
 	else
@@ -1976,7 +1977,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
 		CbObjectWriter Cbo;
 		Cbo.BeginArray("ops");
 		{
-			CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo);
+			CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
 		}
 		Cbo.EndArray();
 		HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index a0dbed741..438943f75 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -694,7 +694,7 @@ ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx)
 
 	using namespace std::literals;
 
-	IterateOplogWithKey([&](int Lsn, const Oid& Key, CbObjectView Op) {
+	IterateOplogWithKey([&](uint32_t Lsn, const Oid& Key, CbObjectView Op) {
 		ZEN_UNUSED(Lsn);
 
 		std::vector<IoHash> Cids;
@@ -1209,7 +1209,7 @@ ProjectStore::Oplog::GetOplogEntryCount() const
 }
 
 void
-ProjectStore::Oplog::IterateOplogWithKey(std::function<void(int, const Oid&, CbObjectView)>&& Handler)
+ProjectStore::Oplog::IterateOplogWithKey(std::function<void(uint32_t, const Oid&, CbObjectView)>&& Handler)
 {
 	RwLock::SharedLockScope _(m_OplogLock);
 	if (!m_Storage)
@@ -1219,7 +1219,7 @@ ProjectStore::Oplog::IterateOplogWithKey(std::function<void(int, const Oid&, CbO
 
 	std::vector<OplogEntryAddress> SortedEntries;
 	std::vector<Oid>			   SortedKeys;
-	std::vector<int>			   SortedLSNs;
+	std::vector<uint32_t>		   SortedLSNs;
 
 	{
 		const auto TargetEntryCount = m_LatestOpMap.size();
@@ -1227,7 +1227,7 @@ ProjectStore::Oplog::IterateOplogWithKey(std::function<void(int, const Oid&, CbO
 		std::vector<size_t>			   EntryIndexes;
 		std::vector<OplogEntryAddress> Entries;
 		std::vector<Oid>			   Keys;
-		std::vector<int>			   LSNs;
+		std::vector<uint32_t>		   LSNs;
 
 		Entries.reserve(TargetEntryCount);
 		EntryIndexes.reserve(TargetEntryCount);
@@ -1270,7 +1270,7 @@ ProjectStore::Oplog::IterateOplogWithKey(std::function<void(int, const Oid&, CbO
 	});
 }
 
-int
+std::optional<uint32_t>
 ProjectStore::Oplog::GetOpIndexByKey(const Oid& Key)
 {
 	RwLock::SharedLockScope _(m_OplogLock);
@@ -1282,18 +1282,7 @@ ProjectStore::Oplog::GetOpIndexByKey(const Oid& Key)
 	{
 		return LatestOp->second;
 	}
-	return -1;
-}
-
-int
-ProjectStore::Oplog::GetMaxOpIndex() const
-{
-	RwLock::SharedLockScope _(m_OplogLock);
-	if (!m_Storage)
-	{
-		return -1;
-	}
-	return gsl::narrow<int>(m_Storage->GetMaxLsn());
+	return {};
 }
 
 std::optional<CbObject>
@@ -1317,7 +1306,7 @@ ProjectStore::Oplog::GetOpByKey(const Oid& Key)
 }
 
 std::optional<CbObject>
-ProjectStore::Oplog::GetOpByIndex(int Index)
+ProjectStore::Oplog::GetOpByIndex(uint32_t Index)
 {
 	RwLock::SharedLockScope _(m_OplogLock);
 	if (!m_Storage)
@@ -1413,7 +1402,7 @@ ProjectStore::Oplog::IterateCapturedLSNs(std::function<bool(const CbObjectView&
 			{
 				return;
 			}
-			for (int UpdatedLSN : *m_CapturedLSNs)
+			for (uint32_t UpdatedLSN : *m_CapturedLSNs)
 			{
 				if (const auto AddressEntryIt = m_OpAddressMap.find(UpdatedLSN); AddressEntryIt != m_OpAddressMap.end())
 				{
@@ -3665,7 +3654,7 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 		// Snapshot all referenced files. This brings the content of all
 		// files into the CID store
 
-		int		 OpCount	  = 0;
+		uint32_t OpCount	  = 0;
 		uint64_t InlinedBytes = 0;
 		uint64_t InlinedFiles = 0;
 		uint64_t TotalBytes	  = 0;
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 98cc29c2a..036c17106 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -94,14 +94,13 @@ public:
 		void				   IterateChunkMap(std::function<void(const Oid&, const IoHash& Hash)>&& Fn);
 		void   IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
 		void   IterateOplog(std::function<void(CbObjectView)>&& Fn);
-		void   IterateOplogWithKey(std::function<void(int, const Oid&, CbObjectView)>&& Fn);
+		void   IterateOplogWithKey(std::function<void(uint32_t, const Oid&, CbObjectView)>&& Fn);
 		void   IterateOplogLocked(std::function<void(CbObjectView)>&& Fn);
 		size_t GetOplogEntryCount() const;
 
 		std::optional<CbObject> GetOpByKey(const Oid& Key);
-		std::optional<CbObject> GetOpByIndex(int Index);
-		int						GetOpIndexByKey(const Oid& Key);
-		int						GetMaxOpIndex() const;
+		std::optional<CbObject> GetOpByIndex(uint32_t Index);
+		std::optional<uint32_t> GetOpIndexByKey(const Oid& Key);
 
 		IoBuffer					 FindChunk(const Oid& ChunkId);
 		IoBuffer					 GetChunkByRawHash(const IoHash& RawHash);
@@ -176,13 +175,13 @@ public:
 		std::filesystem::path m_MarkerPath;
 		std::filesystem::path m_TempPath;
 
-		mutable RwLock						   m_OplogLock;
-		OidMap<IoHash>						   m_ChunkMap;		   // output data chunk id -> CAS address
-		OidMap<IoHash>						   m_MetaMap;		   // meta chunk id -> CAS address
-		OidMap<FileMapEntry>				   m_FileMap;		   // file id -> file map entry
-		int32_t								   m_ManifestVersion;  // File system manifest version
-		tsl::robin_map<int, OplogEntryAddress> m_OpAddressMap;	   // Index LSN -> op data in ops blob file
-		OidMap<int>							   m_LatestOpMap;	   // op key -> latest op LSN for key
+		mutable RwLock								m_OplogLock;
+		OidMap<IoHash>								m_ChunkMap;			// output data chunk id -> CAS address
+		OidMap<IoHash>								m_MetaMap;			// meta chunk id -> CAS address
+		OidMap<FileMapEntry>						m_FileMap;			// file id -> file map entry
+		int32_t										m_ManifestVersion;	// File system manifest version
+		tsl::robin_map<uint32_t, OplogEntryAddress> m_OpAddressMap;		// Index LSN -> op data in ops blob file
+		OidMap<uint32_t>							m_LatestOpMap;		// op key -> latest op LSN for key
 
 		mutable RwLock						   m_UpdateCaptureLock;
 		uint32_t							   m_UpdateCaptureRefCounter = 0;