meta info store (#75)

- Feature: Added option `--gc-cache-attachment-store` which caches referenced attachments in cache records on disk for faster GC - default is `false`
- Feature: Added option `--gc-projectstore-attachment-store` which caches referenced attachments in project store oplogs on disk for faster GC - default is `false`

4 files changed, 161 insertions, 12 deletions

diff --git a/src/zenutil/basicfile.cpp b/src/zenutil/basicfile.cpp
index 266146ca1..174125069 100644
--- a/src/zenutil/basicfile.cpp
+++ b/src/zenutil/basicfile.cpp
@@ -372,7 +372,7 @@ BasicFile::Flush()
 }
 
 uint64_t
-BasicFile::FileSize()
+BasicFile::FileSize() const
 {
 #if ZEN_PLATFORM_WINDOWS
 	ULARGE_INTEGER liFileSize;
@@ -401,7 +401,7 @@ BasicFile::FileSize()
 }
 
 uint64_t
-BasicFile::FileSize(std::error_code& Ec)
+BasicFile::FileSize(std::error_code& Ec) const
 {
 #if ZEN_PLATFORM_WINDOWS
 	ULARGE_INTEGER liFileSize;
@@ -565,20 +565,27 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::
 //////////////////////////////////////////////////////////////////////////
 
 void
-TemporaryFile::SafeWriteFile(std::filesystem::path Path, MemoryView Data)
+TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data)
 {
 	TemporaryFile	TempFile;
 	std::error_code Ec;
-	TempFile.CreateTemporary(Path.parent_path(), Ec);
+	SafeWriteFile(Path, Data, Ec);
 	if (Ec)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to create temp file for file at '{}'", Path));
+		throw std::system_error(Ec, fmt::format("Failed to safely write file '{}'", Path));
 	}
-	TempFile.Write(Data, 0);
-	TempFile.MoveTemporaryIntoPlace(Path, Ec);
-	if (Ec)
+}
+
+void
+TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data, std::error_code& OutEc)
+{
+	TemporaryFile TempFile;
+	if (TempFile.CreateTemporary(Path.parent_path(), OutEc); !OutEc)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", TempFile.GetPath(), Path));
+		if (TempFile.Write(Data, 0, OutEc); !OutEc)
+		{
+			TempFile.MoveTemporaryIntoPlace(Path, OutEc);
+		}
 	}
 }
 
diff --git a/src/zenutil/include/zenutil/basicfile.h b/src/zenutil/include/zenutil/basicfile.h
index 375da20c3..674457196 100644
--- a/src/zenutil/include/zenutil/basicfile.h
+++ b/src/zenutil/include/zenutil/basicfile.h
@@ -60,8 +60,8 @@ public:
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
 	void	 Flush();
-	[[nodiscard]] uint64_t FileSize();
-	[[nodiscard]] uint64_t FileSize(std::error_code& Ec);
+	[[nodiscard]] uint64_t FileSize() const;
+	[[nodiscard]] uint64_t FileSize(std::error_code& Ec) const;
 	void				   SetFileSize(uint64_t FileSize);
 	IoBuffer			   ReadAll();
 	void				   WriteAll(IoBuffer Data, std::error_code& Ec);
@@ -101,7 +101,8 @@ public:
 	void						 MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec);
 	const std::filesystem::path& GetPath() const { return m_TempPath; }
 
-	static void SafeWriteFile(std::filesystem::path Path, MemoryView Data);
+	static void SafeWriteFile(const std::filesystem::path& Path, MemoryView Data);
+	static void SafeWriteFile(const std::filesystem::path& Path, MemoryView Data, std::error_code& OutEc);
 
 private:
 	std::filesystem::path m_TempPath;
diff --git a/src/zenutil/include/zenutil/referencemetadata.h b/src/zenutil/include/zenutil/referencemetadata.h
new file mode 100644
index 000000000..5160bfb8b
--- /dev/null
+++ b/src/zenutil/include/zenutil/referencemetadata.h
@@ -0,0 +1,109 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compositebuffer.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct ReferenceMetaDataHeader
+{
+	static constexpr uint32_t Version1		 = 1;
+	static constexpr uint32_t CurrentVersion = Version1;
+
+	ReferenceMetaDataHeader(uint32_t InMagic, uint32_t InEntryCount, uint64_t InAttachmentCount)
+	: Magic(InMagic)
+	, EntryCount(InEntryCount)
+	, AttachmentCount(InAttachmentCount)
+	, Checksum(ComputeChecksum())
+	{
+	}
+
+	uint32_t Magic	 = 0xffffffffu;
+	uint32_t Version = CurrentVersion;
+
+	uint32_t EntryCount		 = 0;
+	uint64_t AttachmentCount = 0;
+	uint64_t Padding		 = 0;
+
+	uint32_t Checksum = 0;
+
+	bool IsValid(uint32_t ExpectedMagic) const;
+
+	template<typename KeyType, typename AttachmentType>
+	static uint64_t ExpectedSize(uint32_t EntryCount, uint64_t AttachmentCount)
+	{
+		return sizeof(ReferenceMetaDataHeader) + sizeof(KeyType) * EntryCount + sizeof(uint32_t) * EntryCount +
+			   sizeof(AttachmentType) * AttachmentCount;
+	}
+
+	ReferenceMetaDataHeader() = delete;
+
+private:
+	uint32_t ComputeChecksum() const;
+};
+
+static_assert(sizeof(ReferenceMetaDataHeader) == 32);
+
+#pragma pack(pop)
+
+template<typename KeyType, typename AttachmentType>
+CompositeBuffer
+BuildReferenceMetaData(uint32_t						   HeaderMagic,
+					   std::span<const KeyType>		   Keys,
+					   std::span<const uint32_t>	   AttachmentCounts,
+					   std::span<const AttachmentType> Attachments)
+{
+	uint32_t KeyCount = gsl::narrow<uint32_t>(Keys.size());
+
+	ReferenceMetaDataHeader Header(HeaderMagic, KeyCount, Attachments.size());
+	return CompositeBuffer(std::vector<SharedBuffer>{
+		SharedBuffer(IoBuffer(IoBuffer::Clone, &Header, sizeof(ReferenceMetaDataHeader))),
+		SharedBuffer(IoBuffer(IoBuffer::Wrap, Keys.data(), sizeof(KeyType) * Keys.size())),
+		SharedBuffer(IoBuffer(IoBuffer::Wrap, AttachmentCounts.data(), sizeof(uint32_t) * AttachmentCounts.size())),
+		SharedBuffer(IoBuffer(IoBuffer::Wrap, Attachments.data(), sizeof(AttachmentType) * Attachments.size()))});
+}
+
+template<typename KeyType, typename AttachmentType>
+bool
+GetAttachmentsFromMetaData(const IoBuffer&													  MetaData,
+						   const uint32_t													  ExpectedHeaderMagic,
+						   std::function<void(std::span<const KeyType>		  Keys,
+											  std::span<const uint32_t>		  AttachmentCounts,
+											  std::span<const AttachmentType> Attachments)>&& Parser)
+{
+	MemoryView PayloadView = MetaData.GetView();
+	if (PayloadView.GetSize() >= sizeof(ReferenceMetaDataHeader))
+	{
+		const ReferenceMetaDataHeader* Header = (const ReferenceMetaDataHeader*)PayloadView.GetData();
+		if (Header->IsValid(ExpectedHeaderMagic))
+		{
+			if (MetaData.GetSize() ==
+				ReferenceMetaDataHeader::ExpectedSize<KeyType, AttachmentType>(Header->EntryCount, Header->AttachmentCount))
+			{
+				PayloadView.MidInline(sizeof(ReferenceMetaDataHeader));
+				std::span<const KeyType> PayloadKeys = {(const KeyType*)PayloadView.GetData(), Header->EntryCount};
+				PayloadView.MidInline(sizeof(KeyType) * Header->EntryCount);
+				std::span<const uint32_t> PayloadAttachmentCounts = {(const uint32_t*)PayloadView.GetData(), Header->EntryCount};
+				PayloadView.MidInline(sizeof(uint32_t) * Header->EntryCount);
+				std::span<const AttachmentType> PayloadAttachments = {(const AttachmentType*)PayloadView.GetData(),
+																	  Header->AttachmentCount};
+				PayloadView.MidInline(sizeof(AttachmentType) * Header->AttachmentCount);
+				ZEN_ASSERT(PayloadView.GetSize() == 0);
+
+				Parser(PayloadKeys, PayloadAttachmentCounts, PayloadAttachments);
+
+				return true;
+			}
+		}
+	}
+	return false;
+}
+}  // namespace zen
diff --git a/src/zenutil/referencemetadata.cpp b/src/zenutil/referencemetadata.cpp
new file mode 100644
index 000000000..bbcafcfba
--- /dev/null
+++ b/src/zenutil/referencemetadata.cpp
@@ -0,0 +1,32 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/referencemetadata.h>
+//////////////////////////////////////////////////////////////////////////
+
+#include <zencore/xxhash.h>
+
+namespace zen {
+
+uint32_t
+ReferenceMetaDataHeader::ComputeChecksum() const
+{
+	return XXH32(&Magic, sizeof(ReferenceMetaDataHeader) - sizeof(uint32_t), 0xC0C0'BABA);
+}
+
+bool
+ReferenceMetaDataHeader::IsValid(uint32_t ExpectedMagic) const
+{
+	if (Magic != ExpectedMagic)
+	{
+		return false;
+	}
+
+	if (Checksum != ComputeChecksum())
+	{
+		return false;
+	}
+
+	return true;
+}
+
+}  // namespace zen