get oplog attachments (#622)

* add support for downloading individual attachments from an oplog

2 files changed, 108 insertions, 0 deletions

diff --git a/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h b/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h
new file mode 100644
index 000000000..2ddd0f11d
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h
@@ -0,0 +1,106 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/logging.h>
+#include <zencore/uid.h>
+#include <zencore/zencore.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/chunking/chunkedcontent.h>
+#include <zenutil/bufferedwritefilecache.h>
+
+#include <atomic>
+#include <memory>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+class BuildStorageBase;
+class OperationLogOutput;
+struct StorageInstance;
+
+class ProjectStoreOperationOplogState
+{
+public:
+	struct Options
+	{
+		bool				  IsQuiet		= false;
+		bool				  IsVerbose		= false;
+		bool				  ForceDownload = false;
+		std::filesystem::path TempFolderPath;
+	};
+
+	ProjectStoreOperationOplogState(OperationLogOutput& OperationLogOutput,
+									StorageInstance&	Storage,
+									const Oid&			BuildId,
+									const Options&		Options);
+
+	CbObjectView LoadBuildObject();
+	CbObjectView LoadBuildPartsObject();
+	CbObjectView LoadOpsSectionObject();
+	CbArrayView	 LoadBlocksArray();
+	CbArrayView	 LoadChunkedFilesArray();
+	CbArrayView	 LoadChunksArray();
+	CbArray		 LoadArrayFromBuildPart(std::string_view ArrayName);
+
+	const Oid& GetBuildId();
+	const Oid& GetBuildPartId();
+
+private:
+	OperationLogOutput& m_LogOutput;
+	StorageInstance&	m_Storage;
+	const Oid			m_BuildId;
+	const Options		m_Options;
+
+	Oid		 m_BuildPartId;
+	CbObject m_BuildObject;
+	CbObject m_BuildPartsObject;
+	CbObject m_OpsSectionObject;
+	CbArray	 m_BlocksArray;
+	CbArray	 m_ChunkedFilesArray;
+	CbArray	 m_ChunksArray;
+};
+
+class ProjectStoreOperationDownloadAttachments
+{
+public:
+	struct Options
+	{
+		bool				  IsQuiet				= false;
+		bool				  IsVerbose				= false;
+		bool				  ForceDownload			= false;
+		bool				  DecompressAttachments = true;
+		std::filesystem::path TempFolderPath;
+		std::filesystem::path AttachmentOutputPath;
+	};
+
+	ProjectStoreOperationDownloadAttachments(OperationLogOutput&			  OperationLogOutput,
+											 StorageInstance&				  Storage,
+											 std::atomic<bool>&				  AbortFlag,
+											 std::atomic<bool>&				  PauseFlag,
+											 WorkerThreadPool&				  IOWorkerPool,
+											 WorkerThreadPool&				  NetworkPool,
+											 ProjectStoreOperationOplogState& State,
+											 std::span<const IoHash>		  AttachmentHashes,
+											 const Options&					  Options);
+
+	void Execute();
+
+private:
+	OperationLogOutput& m_LogOutput;
+	StorageInstance&	m_Storage;
+	std::atomic<bool>&	m_AbortFlag;
+	std::atomic<bool>&	m_PauseFlag;
+	WorkerThreadPool&	m_IOWorkerPool;
+	WorkerThreadPool&	m_NetworkPool;
+
+	ProjectStoreOperationOplogState&			 m_State;
+	const tsl::robin_set<IoHash, IoHash::Hasher> m_AttachmentHashes;
+	const Options								 m_Options;
+};
+
+}  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
index 4740e7029..de858f818 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
@@ -180,6 +180,8 @@ RemoteProjectStore::Result LoadOplog(CidStore&			  ChunkStore,
 std::vector<IoHash>					   GetBlockHashesFromOplog(CbObjectView ContainerObject);
 std::vector<ThinChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes);
 
+ChunkedInfo ReadChunkedInfo(CbObjectView ChunkedFile);
+
 void remoteprojectstore_forcelink();
 
 }  // namespace zen