Merge branch 'main' of https://github.ol.epicgames.net/ue-foundation/zen into tb/dockertb/docker

7 files changed, 69 insertions, 55 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 69487f9dc..a576ff022 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -482,7 +482,11 @@ BlockStore::TryGetChunk(const BlockStoreLocation& Location) const
 	{
 		if (const Ref<BlockStoreFile>& Block = BlockIt->second; Block)
 		{
-			return Block->GetChunk(Location.Offset, Location.Size);
+			IoBuffer Chunk = Block->GetChunk(Location.Offset, Location.Size);
+			if (Chunk.GetSize() == Location.Size)
+			{
+				return Chunk;
+			}
 		}
 	}
 	return IoBuffer();
@@ -911,7 +915,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 			NewBlockFile = nullptr;
 		}
 	}
-	catch (std::system_error& SystemError)
+	catch (const std::system_error& SystemError)
 	{
 		if (IsOOM(SystemError.code()))
 		{
@@ -926,11 +930,11 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 			ZEN_ERROR("reclaiming space for '{}' failed with system error exception: '{}'", m_BlocksBasePath, SystemError.what());
 		}
 	}
-	catch (std::bad_alloc& BadAlloc)
+	catch (const std::bad_alloc& BadAlloc)
 	{
 		ZEN_WARN("reclaiming space for '{}' ran out of memory: '{}'", m_BlocksBasePath, BadAlloc.what());
 	}
-	catch (std::exception& ex)
+	catch (const std::exception& ex)
 	{
 		ZEN_ERROR("reclaiming space for '{}' failed with: '{}'", m_BlocksBasePath, ex.what());
 	}
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index d897e26ce..f53ab6f8b 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -871,7 +871,7 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(const std::function<uin
 			m_LogFlushPosition = LogCount;
 		}
 	}
-	catch (std::exception& Err)
+	catch (const std::exception& Err)
 	{
 		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
 	}
@@ -1141,7 +1141,7 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc) con
 }
 
 IoBuffer
-ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(ZenContentType ContentType, const IoHash& HashKey) const
+ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const
 {
 	ZEN_TRACE_CPU("Z$::Bucket::GetStandaloneCacheValue");
 
@@ -1152,9 +1152,11 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(ZenContentType ContentTy
 
 	if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.ToPath()))
 	{
-		Data.SetContentType(ContentType);
-
-		return Data;
+		if (Data.GetSize() == Loc.Size())
+		{
+			Data.SetContentType(Loc.GetContentType());
+			return Data;
+		}
 	}
 
 	return {};
@@ -1211,7 +1213,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 		}
 		if (Location.IsFlagSet(DiskLocation::kStandaloneFile))
 		{
-			OutValue.Value = GetStandaloneCacheValue(Location.GetContentType(), HashKey);
+			OutValue.Value = GetStandaloneCacheValue(Location, HashKey);
 		}
 		else
 		{
@@ -1432,7 +1434,7 @@ ZenCacheDiskLayer::CacheBucket::Flush()
 
 		SaveSnapshot();
 	}
-	catch (std::exception& Ex)
+	catch (const std::exception& Ex)
 	{
 		ZEN_WARN("Failed to flush bucket in '{}'. Reason: '{}'", m_BucketDir, Ex.what());
 	}
@@ -1538,7 +1540,7 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 		std::filesystem::path ManifestPath = GetManifestPath(m_BucketDir, m_BucketName);
 		WriteFile(ManifestPath, Buffer);
 	}
-	catch (std::exception& Err)
+	catch (const std::exception& Err)
 	{
 		ZEN_WARN("writing manifest in '{}' FAILED, reason: '{}'", m_BucketDir, Err.what());
 	}
@@ -1621,7 +1623,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 				else
 				{
 					// Structured cache value
-					IoBuffer Buffer = GetStandaloneCacheValue(Loc.GetContentType(), HashKey);
+					IoBuffer Buffer = GetStandaloneCacheValue(Loc, HashKey);
 					if (!Buffer)
 					{
 						ReportBadKey(HashKey);
@@ -1880,7 +1882,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 			IoBuffer Buffer;
 			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 			{
-				if (Buffer = GetStandaloneCacheValue(Loc.GetContentType(), Key); !Buffer)
+				if (Buffer = GetStandaloneCacheValue(Loc, Key); !Buffer)
 				{
 					continue;
 				}
@@ -1983,7 +1985,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 		{
 			SaveSnapshot([&]() { return GcCtx.ClaimGCReserve(); });
 		}
-		catch (std::exception& Ex)
+		catch (const std::exception& Ex)
 		{
 			ZEN_WARN("Failed to write index and manifest after GC in '{}'. Reason: '{}'", m_BucketDir, Ex.what());
 		}
@@ -2269,9 +2271,8 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(RwLock::SharedLockScope& IndexLo
 	const BucketPayload& Payload = m_Payloads[Index];
 	if (Payload.Location.IsFlagSet(DiskLocation::kStructured))
 	{
-		IoBuffer	 Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile)
-								 ? GetStandaloneCacheValue(Payload.Location.GetContentType(), Key)
-								 : GetInlineCacheValue(Payload.Location);
+		IoBuffer	 Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile) ? GetStandaloneCacheValue(Payload.Location, Key)
+																					   : GetInlineCacheValue(Payload.Location);
 		CbObjectView Obj(Value.GetData());
 		Obj.IterateAttachments([&Attachments](CbFieldView Field) { Attachments.emplace_back(Field.AsAttachment()); });
 	}
@@ -2920,7 +2921,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		{
 			SaveSnapshot([]() { return 0; });
 		}
-		catch (std::exception& Ex)
+		catch (const std::exception& Ex)
 		{
 			ZEN_WARN("Failed to write index and manifest after RemoveExpiredData in '{}'. Reason: '{}'", m_BucketDir, Ex.what());
 		}
@@ -3033,7 +3034,7 @@ public:
 			m_IndexLock.reset();
 			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
 		}
-		catch (std::exception& Ex)
+		catch (const std::exception& Ex)
 		{
 			ZEN_ERROR("~DiskBucketReferenceChecker threw exception: '{}'", Ex.what());
 		}
@@ -3066,7 +3067,7 @@ public:
 		{
 			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<HashSet>(); });
 
-			std::vector<IoHash> StandaloneKeys;
+			std::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys;
 			{
 				std::vector<IoHash>						  InlineKeys;
 				std::unordered_map<uint32_t, std::size_t> BlockIndexToEntriesPerBlockIndex;
@@ -3099,7 +3100,7 @@ public:
 						const IoHash& Key = Entry.first;
 						if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 						{
-							StandaloneKeys.push_back(Key);
+							StandaloneKeys.push_back(std::make_pair(Key, Loc));
 							continue;
 						}
 
@@ -3157,7 +3158,7 @@ public:
 					}
 				}
 			}
-			for (const IoHash& Key : StandaloneKeys)
+			for (const auto& It : StandaloneKeys)
 			{
 				if (Ctx.IsCancelledFlag.load())
 				{
@@ -3165,7 +3166,7 @@ public:
 					return;
 				}
 
-				IoBuffer Buffer = m_CacheBucket.GetStandaloneCacheValue(ZenContentType::kCbObject, Key);
+				IoBuffer Buffer = m_CacheBucket.GetStandaloneCacheValue(It.second, It.first);
 				if (!Buffer)
 				{
 					continue;
@@ -3366,7 +3367,7 @@ ZenCacheDiskLayer::~ZenCacheDiskLayer()
 		// This can cause a deadlock, if GC is running we would block while holding ZenCacheDiskLayer::m_Lock
 		m_DroppedBuckets.clear();
 	}
-	catch (std::exception& Ex)
+	catch (const std::exception& Ex)
 	{
 		ZEN_ERROR("~ZenCacheDiskLayer() failed. Reason: '{}'", Ex.what());
 	}
@@ -3490,7 +3491,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 		{
 			IsOk = DeleteDirectories(BadBucketPath);
 		}
-		catch (std::exception&)
+		catch (const std::exception&)
 		{
 		}
 
@@ -3633,14 +3634,14 @@ ZenCacheDiskLayer::Flush()
 					{
 						Bucket->Flush();
 					}
-					catch (std::exception& Ex)
+					catch (const std::exception& Ex)
 					{
 						ZEN_ERROR("Failed flushing bucket. Reason: '{}'", Ex.what());
 					}
 				});
 			}
 		}
-		catch (std::exception& Ex)
+		catch (const std::exception& Ex)
 		{
 			ZEN_ERROR("Failed to flush buckets at '{}'. Reason: '{}'", m_RootDir, Ex.what());
 		}
@@ -3877,7 +3878,7 @@ ZenCacheDiskLayer::MemCacheTrim()
 			}
 		});
 	}
-	catch (std::exception& Ex)
+	catch (const std::exception& Ex)
 	{
 		ZEN_ERROR("Failed scheduling ZenCacheDiskLayer::MemCacheTrim. Reason: '{}'", Ex.what());
 		m_IsMemCacheTrimming.store(false);
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index daa628f77..c4ee6f4d3 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -400,7 +400,7 @@ ZenCacheStore::LogWorker()
 										 ObjectSize,
 										 ToString(Item.Value.Value.GetContentType()))
 						}
-						catch (std::exception& Ex)
+						catch (const std::exception& Ex)
 						{
 							ZEN_LOG_INFO(LogCacheActivity,
 										 "{} [{}] {}/{}/{} failed: Reason: '{}'",
@@ -436,7 +436,7 @@ ZenCacheStore::LogWorker()
 			m_LogEvent.Wait();
 			m_LogEvent.Reset();
 		}
-		catch (std::exception& Ex)
+		catch (const std::exception& Ex)
 		{
 			ZEN_WARN("Log writer failed: '{}'", Ex.what());
 		}
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 17cf20e35..84905df15 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -338,7 +338,7 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 
 		m_BlockStore.IterateChunks(ChunkLocations, ValidateSmallChunk, ValidateLargeChunk);
 	}
-	catch (ScrubDeadlineExpiredException&)
+	catch (const ScrubDeadlineExpiredException&)
 	{
 		ZEN_INFO("Scrubbing deadline expired, operation incomplete");
 	}
@@ -934,7 +934,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 		EntryCount		   = Entries.size();
 		m_LogFlushPosition = IndexLogPosition;
 	}
-	catch (std::exception& Err)
+	catch (const std::exception& Err)
 	{
 		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
 
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 428183827..0f3e2ab5a 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -729,19 +729,28 @@ FileCasStrategy::FindChunk(const IoHash& ChunkHash)
 
 	ZEN_ASSERT(m_IsInitialized);
 
+	uint64_t ExpectedSize = 0;
 	{
 		RwLock::SharedLockScope _(m_Lock);
-		if (!m_Index.contains(ChunkHash))
+		if (auto It = m_Index.find(ChunkHash); It != m_Index.end())
+		{
+			ExpectedSize = It->second.Size;
+		}
+		else
 		{
 			return {};
 		}
 	}
 
-	ShardingHelper Name(m_RootDirectory.c_str(), ChunkHash);
-
+	ShardingHelper			Name(m_RootDirectory.c_str(), ChunkHash);
 	RwLock::SharedLockScope _(LockForHash(ChunkHash));
 
-	return IoBufferBuilder::MakeFromFile(Name.ShardedPath.c_str());
+	if (IoBuffer Chunk = IoBufferBuilder::MakeFromFile(Name.ShardedPath.c_str()); Chunk.GetSize() == ExpectedSize)
+	{
+		return Chunk;
+	}
+
+	return {};
 }
 
 bool
@@ -1201,7 +1210,7 @@ FileCasStrategy::MakeIndexSnapshot()
 		EntryCount		   = Entries.size();
 		m_LogFlushPosition = IndexLogPosition;
 	}
-	catch (std::exception& Err)
+	catch (const std::exception& Err)
 	{
 		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
 
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 1a34019fb..d51144a5a 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -663,7 +663,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 								StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats);
 							}
 						}
-						catch (std::exception& Ex)
+						catch (const std::exception& Ex)
 						{
 							ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
 						}
@@ -733,7 +733,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
 									}
 								}
-								catch (std::exception& Ex)
+								catch (const std::exception& Ex)
 								{
 									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
 											  ReferenceStore->GetGcName(Ctx),
@@ -806,7 +806,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 											}
 										}
 									}
-									catch (std::exception& Ex)
+									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
 												  Referencer->GetGcName(Ctx),
@@ -863,7 +863,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 										Checker->PreCache(Ctx);
 									}
-									catch (std::exception& Ex)
+									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
 									}
@@ -919,7 +919,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										SCOPED_TIMER(Stats->second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 										Checker->LockState(Ctx);
 									}
-									catch (std::exception& Ex)
+									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed locking state for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
 									}
@@ -997,7 +997,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 												StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->CompactStoreStats);
 											}
 										}
-										catch (std::exception& Ex)
+										catch (const std::exception& Ex)
 										{
 											ZEN_ERROR("GCV2: Failed locking state for {}. Reason: '{}'", Pruner->GetGcName(Ctx), Ex.what());
 										}
@@ -1563,7 +1563,7 @@ GcScheduler::AppendGCLog(GcClock::TimePoint StartTime, const GcSettings& Setting
 			GcLogFile.Write(EntryBuffer, AppendPos);
 		}
 	}
-	catch (std::system_error& SystemError)
+	catch (const std::system_error& SystemError)
 	{
 		if (IsOOM(SystemError.code()))
 		{
@@ -1578,11 +1578,11 @@ GcScheduler::AppendGCLog(GcClock::TimePoint StartTime, const GcSettings& Setting
 			ZEN_ERROR("writing gc result failed with system error exception: '{}'", SystemError.what());
 		}
 	}
-	catch (std::bad_alloc& BadAlloc)
+	catch (const std::bad_alloc& BadAlloc)
 	{
 		ZEN_WARN("writing gc result ran out of memory: '{}'", BadAlloc.what());
 	}
-	catch (std::exception& Ex)
+	catch (const std::exception& Ex)
 	{
 		ZEN_ERROR("writing gc result failed with: '{}'", Ex.what());
 	}
@@ -1970,7 +1970,7 @@ GcScheduler::SchedulerThread()
 
 			WaitTime = std::chrono::seconds(0);
 		}
-		catch (std::system_error& SystemError)
+		catch (const std::system_error& SystemError)
 		{
 			if (IsOOM(SystemError.code()))
 			{
@@ -1988,14 +1988,14 @@ GcScheduler::SchedulerThread()
 			m_LastLightweightGcTime = m_LastGcTime;
 			WaitTime				= m_Config.MonitorInterval;
 		}
-		catch (std::bad_alloc& BadAlloc)
+		catch (const std::bad_alloc& BadAlloc)
 		{
 			ZEN_WARN("scheduling garbage collection ran out of memory: '{}'", BadAlloc.what());
 			m_LastGcTime			= GcClock::Now();
 			m_LastLightweightGcTime = m_LastGcTime;
 			WaitTime				= m_Config.MonitorInterval;
 		}
-		catch (std::exception& Ex)
+		catch (const std::exception& Ex)
 		{
 			ZEN_ERROR("scheduling garbage collection failed with: '{}'", Ex.what());
 			m_LastGcTime			= GcClock::Now();
@@ -2028,7 +2028,7 @@ GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds Time
 		Ctx.SetShouldDelete(DoDelete);
 		m_GcManager.ScrubStorage(Ctx);
 	}
-	catch (ScrubDeadlineExpiredException&)
+	catch (const ScrubDeadlineExpiredException&)
 	{
 		ZEN_INFO("scrubbing deadline expired (top level), operation incomplete!");
 	}
@@ -2189,7 +2189,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 			SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
 			SaveCompactBinaryObject(Path, SchedulerState.Save());
 		}
-		catch (std::system_error& SystemError)
+		catch (const std::system_error& SystemError)
 		{
 			if (IsOOM(SystemError.code()))
 			{
@@ -2204,11 +2204,11 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 				ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}'", SystemError.what());
 			}
 		}
-		catch (std::bad_alloc& BadAlloc)
+		catch (const std::bad_alloc& BadAlloc)
 		{
 			ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", BadAlloc.what());
 		}
-		catch (std::exception& Ex)
+		catch (const std::exception& Ex)
 		{
 			ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what());
 		}
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 7aced67ad..471cc5dcd 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -326,7 +326,7 @@ public:
 
 		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
 		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
-		IoBuffer GetStandaloneCacheValue(ZenContentType ContentType, const IoHash& HashKey) const;
+		IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const;
 		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
 		CacheValueDetails::ValueDetails GetValueDetails(RwLock::SharedLockScope&, const IoHash& Key, PayloadIndex Index) const;