Merge branch 'main' of https://github.com/EpicGames/zen

14 files changed, 274 insertions, 48 deletions

diff --git a/zen/cmds/print.cpp b/zen/cmds/print.cpp
new file mode 100644
index 000000000..aac6afd44
--- /dev/null
+++ b/zen/cmds/print.cpp
@@ -0,0 +1,107 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "print.h"
+
+#include <zencore/compactbinarypackage.h>
+#include <zencore/filesystem.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+
+using namespace std::literals;
+
+namespace zen {
+
+PrintCommand::PrintCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "s", "source", "Object payload file", cxxopts::value(m_Filename), "<file name>");
+}
+
+PrintCommand::~PrintCommand() = default;
+
+int
+PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions, argc, argv);
+
+	m_Options.parse_positional({"source"});
+
+	auto result = m_Options.parse(argc, argv);
+
+	if (result.count("help"))
+	{
+		std::cout << m_Options.help({"", "Group"}) << std::endl;
+
+		return 0;
+	}
+
+	// Validate arguments
+
+	if (m_Filename.empty())
+		throw std::runtime_error("No file specified");
+
+	zen::FileContents Fc   = zen::ReadFile(m_Filename);
+	IoBuffer		  Data = Fc.Flatten();
+	zen::CbObject	  Object{SharedBuffer(Data)};
+
+	zen::StringBuilder<1024> ObjStr;
+	zen::CompactBinaryToJson(Object, ObjStr);
+	zen::ConsoleLog().info("{}", ObjStr);
+
+	return 0;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+PrintPackageCommand::PrintPackageCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "s", "source", "Package payload file", cxxopts::value(m_Filename), "<file name>");
+}
+
+PrintPackageCommand::~PrintPackageCommand()
+{
+}
+
+int
+PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions, argc, argv);
+
+	m_Options.parse_positional({"source"});
+
+	auto result = m_Options.parse(argc, argv);
+
+	if (result.count("help"))
+	{
+		std::cout << m_Options.help({"", "Group"}) << std::endl;
+
+		return 0;
+	}
+
+	// Validate arguments
+
+	if (m_Filename.empty())
+		throw std::runtime_error("No file specified");
+
+	zen::FileContents Fc   = zen::ReadFile(m_Filename);
+	IoBuffer		  Data = Fc.Flatten();
+	zen::CbPackage	  Package;
+
+	bool Ok = Package.TryLoad(Data) || zen::legacy::TryLoadCbPackage(Package, Data, &UniqueBuffer::Alloc);
+
+	if (Ok)
+	{
+		zen::StringBuilder<1024> ObjStr;
+		zen::CompactBinaryToJson(Package.GetObject(), ObjStr);
+		zen::ConsoleLog().info("{}", ObjStr);
+	}
+	else
+	{
+		zen::ConsoleLog().error("error: malformed package?");
+	}
+
+	return 0;
+}
+
+}  // namespace zen
diff --git a/zen/cmds/print.h b/zen/cmds/print.h
new file mode 100644
index 000000000..eed0aa14e
--- /dev/null
+++ b/zen/cmds/print.h
@@ -0,0 +1,41 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+/** Print Compact Binary
+ */
+class PrintCommand : public ZenCmdBase
+{
+public:
+	PrintCommand();
+	~PrintCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options* Options() override { return &m_Options; }
+
+private:
+	cxxopts::Options m_Options{"print", "Print compact binary object"};
+	std::string		 m_Filename;
+};
+
+/** Print Compact Binary Package
+ */
+class PrintPackageCommand : public ZenCmdBase
+{
+public:
+	PrintPackageCommand();
+	~PrintPackageCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options* Options() override { return &m_Options; }
+
+private:
+	cxxopts::Options m_Options{"printpkg", "Print compact binary package"};
+	std::string		 m_Filename;
+};
+
+}  // namespace zen
diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp
index 94eb7ef6d..19b5c8980 100644
--- a/zen/cmds/run.cpp
+++ b/zen/cmds/run.cpp
@@ -10,6 +10,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/iohash.h>
 #include <zencore/logging.h>
+#include <zencore/stream.h>
 #include <zencore/string.h>
 #include <zencore/timer.h>
 #include <zenutil/zenserverprocess.h>
diff --git a/zen/zen.cpp b/zen/zen.cpp
index 86c41d658..3c33ff5e0 100644
--- a/zen/zen.cpp
+++ b/zen/zen.cpp
@@ -9,6 +9,7 @@
 #include "cmds/dedup.h"
 #include "cmds/deploy.h"
 #include "cmds/hash.h"
+#include "cmds/print.h"
 #include "cmds/run.h"
 #include "cmds/status.h"
 #include "cmds/top.h"
@@ -98,18 +99,20 @@ main(int argc, char** argv)
 
 	auto _ = zen::MakeGuard([] { spdlog::shutdown(); });
 
-	HashCommand	  HashCmd;
-	CopyCommand	  CopyCmd;
-	DedupCommand  DedupCmd;
-	DeployCommand DeployCmd;
-	DropCommand	  DropCmd;
-	ChunkCommand  ChunkCmd;
-	RunCommand	  RunCmd;
-	StatusCommand StatusCmd;
-	TopCommand	  TopCmd;
-	PsCommand	  PsCmd;
-	UpCommand	  UpCmd;
-	DownCommand	  DownCmd;
+	HashCommand			HashCmd;
+	CopyCommand			CopyCmd;
+	DedupCommand		DedupCmd;
+	DeployCommand		DeployCmd;
+	DropCommand			DropCmd;
+	ChunkCommand		ChunkCmd;
+	RunCommand			RunCmd;
+	StatusCommand		StatusCmd;
+	TopCommand			TopCmd;
+	PrintCommand		PrintCmd;
+	PrintPackageCommand PrintPkgCmd;
+	PsCommand			PsCmd;
+	UpCommand			UpCmd;
+	DownCommand			DownCmd;
 
 #if ZEN_WITH_TESTS
 	RunTestsCommand RunTestsCmd;
@@ -128,6 +131,8 @@ main(int argc, char** argv)
 		{"dedup",		&DedupCmd,		"Dedup files"},
 		{"drop",		&DropCmd,		"Drop cache bucket(s)"},
 		{"hash",		&HashCmd,		"Compute file hashes"},
+		{"print",		&PrintCmd,		"Print compact binary object"},
+		{"printpackage", &PrintPkgCmd,	"Print compact binary package"},
 		{"run",			&RunCmd,		"Remote execution"},
 		{"status",		&StatusCmd,		"Show zen status"},
 		{"ps",			&PsCmd,			"Enumerate running zen server instances"},
diff --git a/zen/zen.vcxproj b/zen/zen.vcxproj
index fb0674e87..f31c0bc17 100644
--- a/zen/zen.vcxproj
+++ b/zen/zen.vcxproj
@@ -99,6 +99,7 @@
     <ClCompile Include="cmds\dedup.cpp" />
     <ClCompile Include="cmds\deploy.cpp" />
     <ClCompile Include="cmds\hash.cpp" />
+    <ClCompile Include="cmds\print.cpp" />
     <ClCompile Include="cmds\run.cpp" />
     <ClCompile Include="cmds\scrub.cpp" />
     <ClCompile Include="cmds\status.cpp" />
@@ -114,6 +115,7 @@
     <ClInclude Include="cmds\dedup.h" />
     <ClInclude Include="cmds\deploy.h" />
     <ClInclude Include="cmds\hash.h" />
+    <ClInclude Include="cmds\print.h" />
     <ClInclude Include="cmds\run.h" />
     <ClInclude Include="cmds\scrub.h" />
     <ClInclude Include="cmds\status.h" />
diff --git a/zen/zen.vcxproj.filters b/zen/zen.vcxproj.filters
index 9002f01c2..d983b413c 100644
--- a/zen/zen.vcxproj.filters
+++ b/zen/zen.vcxproj.filters
@@ -28,6 +28,7 @@
     <ClCompile Include="cmds\up.cpp" />
     <ClCompile Include="cmds\cache.cpp" />
     <ClCompile Include="cmds\scrub.cpp" />
+    <ClCompile Include="cmds\print.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="chunk\chunk.h" />
@@ -57,6 +58,7 @@
     <ClInclude Include="cmds\up.h" />
     <ClInclude Include="cmds\cache.h" />
     <ClInclude Include="cmds\scrub.h" />
+    <ClInclude Include="cmds\print.h" />
   </ItemGroup>
   <ItemGroup>
     <Filter Include="cmds">
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index 1d6b2f61c..f6ba92f98 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -522,6 +522,23 @@ WriteFile(std::filesystem::path Path, IoBuffer Data)
 	WriteFile(Path, &DataPtr, 1);
 }
 
+IoBuffer 
+FileContents::Flatten()
+{
+	if (Data.size() == 1)
+	{
+		return Data[0];
+	}
+	else if (Data.empty())
+	{
+		return {};
+	}
+	else
+	{
+		ZEN_NOT_IMPLEMENTED();
+	}
+}
+
 FileContents
 ReadFile(std::filesystem::path Path)
 {
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index 6678528f6..c7ac7140d 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -2,9 +2,10 @@
 
 #pragma once
 
-#include "stream.h"
 #include "zencore.h"
 
+#include <zencore/iobuffer.h>
+
 #include <filesystem>
 #include <functional>
 
@@ -36,6 +37,8 @@ struct FileContents
 {
 	std::vector<IoBuffer> Data;
 	std::error_code		  ErrorCode;
+
+	IoBuffer Flatten();
 };
 
 ZENCORE_API FileContents ReadFile(std::filesystem::path Path);
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index ada19216c..8ab0276c5 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -676,11 +676,11 @@ HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request
 		case kHead:
 		case kGet:
 			{
-				HandleGetCachePayload(Request, Ref, Policy);
 				if (Verb == kHead)
 				{
 					Request.SetSuppressResponseBody();
 				}
+				HandleGetCachePayload(Request, Ref, Policy);
 			}
 			break;
 		case kPut:
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 5e93ebaa9..b97f0830f 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -116,6 +116,13 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 	m_DiskLayer.Scrub(Ctx);
 	m_MemLayer.Scrub(Ctx);
 }
+
+void
+ZenCacheStore::GarbageCollect(GcContext& GcCtx)
+{
+	ZEN_UNUSED(GcCtx);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 ZenCacheMemoryLayer::ZenCacheMemoryLayer()
@@ -195,6 +202,12 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
 }
 
 void
+ZenCacheMemoryLayer::GarbageCollect(GcContext& GcCtx)
+{
+	ZEN_UNUSED(GcCtx);
+}
+
+void
 ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 {
 	std::vector<IoHash> BadHashes;
@@ -294,6 +307,7 @@ struct ZenCacheDiskLayer::CacheBucket
 	void Drop();
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
+	void GarbageCollect(GcContext& GcCtx);
 
 	inline bool IsOk() const { return m_Ok; }
 
@@ -611,6 +625,12 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 }
 
 void
+ZenCacheDiskLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
+{
+	ZEN_UNUSED(GcCtx);
+}
+
+void
 ZenCacheDiskLayer::CacheBucket::PutLargeObject(const IoHash& HashKey, const ZenCacheValue& Value)
 {
 	WideStringBuilder<128> DataFilePath;
@@ -830,27 +850,10 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
 	}
 }
 
-//////////////////////////////////////////////////////////////////////////
-
-ZenCacheTracker::ZenCacheTracker(ZenCacheStore& CacheStore)
-{
-	ZEN_UNUSED(CacheStore);
-}
-
-ZenCacheTracker::~ZenCacheTracker()
-{
-}
-
-void
-ZenCacheTracker::TrackAccess(std::string_view Bucket, const IoHash& HashKey)
-{
-	ZEN_UNUSED(Bucket);
-	ZEN_UNUSED(HashKey);
-}
-
 void
-ZenCacheTracker::Flush()
+ZenCacheDiskLayer::GarbageCollect(GcContext& GcCtx)
 {
+	ZEN_UNUSED(GcCtx);
 }
 
 }  // namespace zen
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index f96757409..011f13323 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -56,6 +56,7 @@ public:
 	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
 	bool DropBucket(std::string_view Bucket);
 	void Scrub(ScrubContext& Ctx);
+	void GarbageCollect(GcContext& GcCtx);
 
 private:
 	struct CacheBucket
@@ -83,6 +84,7 @@ public:
 	bool DropBucket(std::string_view Bucket);
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
+	void GarbageCollect(GcContext& GcCtx);
 
 private:
 	/** A cache bucket manages a single directory containing
@@ -107,6 +109,7 @@ public:
 	bool DropBucket(std::string_view Bucket);
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
+	void GarbageCollect(GcContext& GcCtx);
 
 private:
 	std::filesystem::path m_RootDir;
@@ -116,18 +119,4 @@ private:
 	uint64_t			  m_LastScrubTime		   = 0;
 };
 
-/** Tracks cache entry access, stats and orchestrates cleanup activities
- */
-class ZenCacheTracker
-{
-public:
-	ZenCacheTracker(ZenCacheStore& CacheStore);
-	~ZenCacheTracker();
-
-	void TrackAccess(std::string_view Bucket, const IoHash& HashKey);
-	void Flush();
-
-private:
-};
-
 }  // namespace zen
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 7870f9559..6b24692e1 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -1425,7 +1425,12 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
 
 			if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver))
 			{
-				ZEN_ERROR("Received malformed package!");
+				std::filesystem::path BadPackagePath =
+					Oplog.TempPath() / "bad_packages" / "session{}_request{}"_format(HttpReq.SessionId(), HttpReq.RequestId());
+
+				ZEN_ERROR("Received malformed package! Saving payload to '{}'", BadPackagePath);
+
+				zen::WriteFile(BadPackagePath, Payload);
 
 				return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package");
 			}
diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp
index 1db2b50bf..808fc8fb3 100644
--- a/zenstore/CAS.cpp
+++ b/zenstore/CAS.cpp
@@ -32,6 +32,15 @@ CasChunkSet::AddChunkToSet(const IoHash& HashToAdd)
 }
 
 void
+CasChunkSet::AddChunksToSet(std::span<const IoHash> HashesToAdd)
+{
+	for (const IoHash& Hash : HashesToAdd)
+	{
+		m_ChunkSet.insert(Hash);
+	}
+}
+
+void
 CasChunkSet::RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate)
 {
 	for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd;)
@@ -58,6 +67,34 @@ CasChunkSet::IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callba
 
 //////////////////////////////////////////////////////////////////////////
 
+struct GcContext::GcState
+{
+	CasChunkSet m_CasChunks;
+	CasChunkSet m_CidChunks;
+};
+
+GcContext::GcContext() : m_State(std::make_unique<GcState>())
+{
+}
+
+GcContext::~GcContext()
+{
+}
+
+void
+GcContext::ContributeCids(std::span<const IoHash> Cids)
+{
+	m_State->m_CidChunks.AddChunksToSet(Cids);
+}
+
+void
+GcContext::ContributeCas(std::span<const IoHash> Cas)
+{
+	m_State->m_CasChunks.AddChunksToSet(Cas);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 void
 ScrubContext::ReportBadChunks(std::span<IoHash> BadChunks)
 {
diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h
index 93454ca6f..1425845a0 100644
--- a/zenstore/include/zenstore/CAS.h
+++ b/zenstore/include/zenstore/CAS.h
@@ -37,7 +37,16 @@ struct CasStoreConfiguration
 class GcContext
 {
 public:
+	GcContext();
+	~GcContext();
+
+	void ContributeCids(std::span<const IoHash> Cid);
+	void ContributeCas(std::span<const IoHash> Hash);
+
 private:
+	struct GcState;
+
+	std::unique_ptr<GcState> m_State;
 };
 
 /** Context object for data scrubbing
@@ -58,10 +67,14 @@ private:
 	bool	 m_Recover	 = true;
 };
 
+/** Manage a set of IoHash values
+ */
+
 class CasChunkSet
 {
 public:
 	void						AddChunkToSet(const IoHash& HashToAdd);
+	void						AddChunksToSet(std::span<const IoHash> HashesToAdd);
 	void						RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate);
 	void						IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback);
 	inline [[nodiscard]] bool	ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); }
@@ -69,6 +82,7 @@ public:
 	inline [[nodiscard]] size_t GetSize() const { return m_ChunkSet.size(); }
 
 private:
+	// Q: should we protect this with a lock, or is that a higher level concern?
 	std::unordered_set<IoHash> m_ChunkSet;
 };