reduced memory churn using fixed_xxx containers (#236)

* Added EASTL to help with eliminating memory allocations
* Applied EASTL to eliminate memory allocations, primarily by using `fixed_vector` et al to use stack allocations / inline struct allocations

Reduces memory events in traces by close to a factor of 10 in test scenario (starting editor for project F)

2 files changed, 49 insertions, 28 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 0b8e5f13b..47748dd90 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -983,15 +983,19 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
 	IoBuffer Payload	   = HttpReq.ReadPayload();
 	CbObject RequestObject = LoadCompactBinaryObject(Payload);
 
-	std::vector<IoHash> ChunkList;
-	CbArrayView			HaveList = RequestObject["have"sv].AsArrayView();
-	ChunkList.reserve(HaveList.Num());
-	for (auto& Entry : HaveList)
+	std::vector<IoHash> NeedList;
+
 	{
-		ChunkList.push_back(Entry.AsHash());
-	}
+		eastl::fixed_vector<IoHash, 16> ChunkList;
+		CbArrayView						HaveList = RequestObject["have"sv].AsArrayView();
+		ChunkList.reserve(HaveList.Num());
+		for (auto& Entry : HaveList)
+		{
+			ChunkList.push_back(Entry.AsHash());
+		}
 
-	std::vector<IoHash> NeedList = FoundLog->CheckPendingChunkReferences(ChunkList, std::chrono::minutes(2));
+		NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2));
+	}
 
 	CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1);
 	Cbo.BeginArray("need");
@@ -1151,7 +1155,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
 	}
 
-	std::vector<IoHash> ReferencedChunks;
+	eastl::fixed_vector<IoHash, 16> ReferencedChunks;
 	Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); });
 
 	// Write core to oplog
@@ -1169,7 +1173,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	// Once we stored the op, we no longer need to retain any chunks this op references
 	if (!ReferencedChunks.empty())
 	{
-		FoundLog->RemovePendingChunkReferences(ReferencedChunks);
+		FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks)));
 	}
 
 	m_ProjectStats.OpWriteCount++;
@@ -1301,9 +1305,9 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 
 	HttpServerRequest& HttpReq = Req.ServerRequest();
 
-	const std::string& ProjectId  = Req.GetCapture(1);
-	const std::string& OplogId	  = Req.GetCapture(2);
-	const std::string& OpIdString = Req.GetCapture(3);
+	const std::string_view ProjectId  = Req.GetCapture(1);
+	const std::string_view OplogId	  = Req.GetCapture(2);
+	const std::string_view OpIdString = Req.GetCapture(3);
 
 	Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
 	if (!Project)
@@ -1690,8 +1694,8 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req)
 
 	using namespace std::literals;
 
-	HttpServerRequest& HttpReq	 = Req.ServerRequest();
-	const std::string  ProjectId = Req.GetCapture(1);
+	HttpServerRequest&	   HttpReq	 = Req.ServerRequest();
+	const std::string_view ProjectId = Req.GetCapture(1);
 
 	switch (HttpReq.RequestVerb())
 	{
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 53df12b14..86791e29a 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -423,9 +423,13 @@ ComputeOpKey(const CbObjectView& Op)
 {
 	using namespace std::literals;
 
-	BinaryWriter KeyStream;
+	eastl::fixed_vector<uint8_t, 256> KeyData;
 
-	Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyStream.Write(Data, Size); });
+	Op["key"sv].WriteToStream([&](const void* Data, size_t Size) {
+		auto Begin = reinterpret_cast<const uint8_t*>(Data);
+		auto End   = Begin + Size;
+		KeyData.insert(KeyData.end(), Begin, End);
+	});
 
 	XXH3_128 KeyHash128;
 
@@ -434,15 +438,15 @@ ComputeOpKey(const CbObjectView& Op)
 	// path but longer paths are evaluated properly. In the future all key lengths
 	// should be evaluated using the proper path, this is a temporary workaround to
 	// maintain compatibility with existing disk state.
-	if (KeyStream.GetSize() < 240)
+	if (KeyData.size() < 240)
 	{
 		XXH3_128Stream_deprecated KeyHasher;
-		KeyHasher.Append(KeyStream.Data(), KeyStream.Size());
+		KeyHasher.Append(KeyData.data(), KeyData.size());
 		KeyHash128 = KeyHasher.GetHash();
 	}
 	else
 	{
-		KeyHash128 = XXH3_128::HashMemory(KeyStream.GetView());
+		KeyHash128 = XXH3_128::HashMemory(KeyData.data(), KeyData.size());
 	}
 
 	Oid KeyHash;
@@ -2735,7 +2739,7 @@ ProjectStore::Oplog::CheckPendingChunkReferences(std::span<const IoHash> ChunkHa
 	MissingChunks.reserve(ChunkHashes.size());
 	for (const IoHash& FileHash : ChunkHashes)
 	{
-		if (IoBuffer Payload = m_CidStore.FindChunkByCid(FileHash); !Payload)
+		if (!m_CidStore.ContainsChunk(FileHash))
 		{
 			MissingChunks.push_back(FileHash);
 		}
@@ -3359,7 +3363,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo
 	ZEN_MEMSCOPE(GetProjectstoreTag());
 	ZEN_TRACE_CPU("Store::OpenOplog");
 
-	std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
 	{
 		RwLock::SharedLockScope ProjectLock(m_ProjectLock);
 
@@ -3367,21 +3370,35 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo
 
 		if (OplogIt != m_Oplogs.end())
 		{
-			if (!VerifyPathOnDisk || Oplog::ExistsAt(OplogBasePath))
+			bool ReOpen = false;
+
+			if (VerifyPathOnDisk)
 			{
-				return OplogIt->second.get();
+				std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
+
+				if (!Oplog::ExistsAt(OplogBasePath))
+				{
+					// Somebody deleted the oplog on disk behind our back
+					ProjectLock.ReleaseNow();
+					std::filesystem::path DeletePath;
+					if (!RemoveOplog(OplogId, DeletePath))
+					{
+						ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+					}
+
+					ReOpen = true;
+				}
 			}
 
-			// Somebody deleted the oplog on disk behind our back
-			ProjectLock.ReleaseNow();
-			std::filesystem::path DeletePath;
-			if (!RemoveOplog(OplogId, DeletePath))
+			if (!ReOpen)
 			{
-				ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+				return OplogIt->second.get();
 			}
 		}
 	}
 
+	std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
+
 	RwLock::ExclusiveLockScope Lock(m_ProjectLock);
 	if (auto It = m_Oplogs.find(std::string{OplogId}); It != m_Oplogs.end())
 	{