structured compute basics (#714)

this change adds the `zencompute` component, which can be used to distribute work dispatched from UE using the DDB (Derived Data Build) APIs via zenserver

this change also adds a distinct zenserver compute mode (`zenserver compute`) which is intended to be used for leaf compute nodes

to exercise the compute functionality without directly involving UE, a `zen exec` subcommand is also added, which can be used to feed replays through the system

all new functionality is considered *experimental* and disabled by default at this time, behind the `zencompute` option in xmake config

1 files changed, 91 insertions, 0 deletions

diff --git a/src/zencompute/actionrecorder.h b/src/zencompute/actionrecorder.h
new file mode 100644
index 000000000..9cc2b44a2
--- /dev/null
+++ b/src/zencompute/actionrecorder.h
@@ -0,0 +1,91 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencompute/functionservice.h>
+#include <zencompute/zencompute.h>
+#include <zencore/basicfile.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zenstore/cidstore.h>
+#include <zenstore/gc.h>
+#include <zenstore/zenstore.h>
+
+#include <filesystem>
+#include <functional>
+#include <map>
+#include <unordered_map>
+
+namespace zen {
+class CbObject;
+class CbPackage;
+struct IoHash;
+}  // namespace zen
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+namespace zen::compute {
+
+//////////////////////////////////////////////////////////////////////////
+
+struct RecordingFileWriter
+{
+	RecordingFileWriter(RecordingFileWriter&&) = delete;
+	RecordingFileWriter& operator=(RecordingFileWriter&&) = delete;
+
+	RwLock		   m_FileLock;
+	BasicFile	   m_File;
+	uint64_t	   m_FileOffset = 0;
+	CbObjectWriter m_TocWriter;
+	BasicFile	   m_TocFile;
+
+	RecordingFileWriter();
+	~RecordingFileWriter();
+
+	void Open(std::filesystem::path FilePath);
+	void Close();
+	void AppendObject(const CbObject& Object, const IoHash& ObjectHash);
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+/**
+ * Recording "runner" implementation
+ *
+ * This class writes out all actions and their attachments to a recording directory
+ * in a format that can be read back by the RecordingReader.
+ *
+ * The contents of the recording directory will be self-contained, with all referenced
+ * attachments stored in the recording directory itself, so that the recording can be
+ * moved or shared without needing to maintain references to the main CID store.
+ *
+ */
+
+class ActionRecorder
+{
+public:
+	ActionRecorder(ChunkResolver& InChunkResolver, const std::filesystem::path& RecordingLogPath);
+	~ActionRecorder();
+
+	ActionRecorder(const ActionRecorder&) = delete;
+	ActionRecorder& operator=(const ActionRecorder&) = delete;
+
+	void Shutdown();
+	void RegisterWorker(const CbPackage& WorkerPackage);
+	bool RecordAction(Ref<RunnerAction> Action);
+
+private:
+	ChunkResolver&		  m_ChunkResolver;
+	std::filesystem::path m_RecordingLogDir;
+
+	RecordingFileWriter	  m_WorkersFile;
+	RecordingFileWriter	  m_ActionsFile;
+	GcManager			  m_Gc;
+	CidStore			  m_CidStore{m_Gc};
+	std::atomic<int>	  m_ChunkCounter{0};
+	std::atomic<uint64_t> m_ChunkBytesCounter{0};
+	std::atomic<int>	  m_ActionsCounter{0};
+};
+
+}  // namespace zen::compute
+
+#endif	// ZEN_WITH_COMPUTE_SERVICES