From a0a0dba13317533f882a85b7f4087588cfa09066 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Thu, 27 Mar 2025 14:09:01 +0100
Subject: optional compress of block chunks (#326)

- Feature: zenserver: Add command line option `--gc-buildstore-duration-seconds` to control GC life time of build store data
 - Improvement: ELF and MachO executable files are no longer chunked
- Improvement: Compress chunks in blocks that encloses a full file (such as small executables)
- Bugfix: Strip path delimiter at end of string in StringToPath
---
 src/zenutil/chunkingcontroller.cpp | 39 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

(limited to 'src/zenutil/chunkingcontroller.cpp')

diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp
index 2a7057a46..a5ebce193 100644
--- a/src/zenutil/chunkingcontroller.cpp
+++ b/src/zenutil/chunkingcontroller.cpp
@@ -41,9 +41,13 @@ class BasicChunkingController : public ChunkingController
 {
 public:
 	BasicChunkingController(std::span<const std::string_view> ExcludeExtensions,
+							bool							  ExcludeElfFiles,
+							bool							  ExcludeMachOFiles,
 							uint64_t						  ChunkFileSizeLimit,
 							const ChunkedParams&			  ChunkingParams)
 	: m_ChunkExcludeExtensions(ExcludeExtensions.begin(), ExcludeExtensions.end())
+	, m_ExcludeElfFiles(ExcludeElfFiles)
+	, m_ExcludeMachOFiles(ExcludeMachOFiles)
 	, m_ChunkFileSizeLimit(ChunkFileSizeLimit)
 	, m_ChunkingParams(ChunkingParams)
 	{
@@ -51,6 +55,8 @@ public:
 
 	BasicChunkingController(CbObjectView Parameters)
 	: m_ChunkExcludeExtensions(ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()))
+	, m_ExcludeElfFiles(Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles))
+	, m_ExcludeMachOFiles(Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles))
 	, m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit))
 	, m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView()))
 	{
@@ -73,6 +79,25 @@ public:
 		}
 
 		BasicFile Buffer(InputPath, BasicFile::Mode::kRead);
+		if (m_ExcludeElfFiles && Buffer.FileSize() > 4)
+		{
+			uint32_t ElfCheck = 0;
+			Buffer.Read(&ElfCheck, 4, 0);
+			if (ElfCheck == 0x464c457f)
+			{
+				return false;
+			}
+		}
+		if (m_ExcludeMachOFiles && Buffer.FileSize() > 4)
+		{
+			uint32_t MachOCheck = 0;
+			Buffer.Read(&MachOCheck, 4, 0);
+			if ((MachOCheck == 0xfeedface) || (MachOCheck == 0xcefaedfe))
+			{
+				return false;
+			}
+		}
+
 		OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed, &AbortFlag);
 		return true;
 	}
@@ -90,6 +115,10 @@ public:
 			}
 		}
 		Writer.EndArray();	// ChunkExcludeExtensions
+
+		Writer.AddBool("ExcludeElfFiles"sv, m_ExcludeElfFiles);
+		Writer.AddBool("ExcludeMachOFiles"sv, m_ExcludeMachOFiles);
+
 		Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit);
 		Writer.BeginObject("ChunkingParams"sv);
 		{
@@ -106,6 +135,8 @@ public:
 
 protected:
 	const std::vector<std::string> m_ChunkExcludeExtensions;
+	const bool					   m_ExcludeElfFiles   = false;
+	const bool					   m_ExcludeMachOFiles = false;
 	const uint64_t				   m_ChunkFileSizeLimit;
 	const ChunkedParams			   m_ChunkingParams;
 };
@@ -230,10 +261,16 @@ protected:
 
 std::unique_ptr<ChunkingController>
 CreateBasicChunkingController(std::span<const std::string_view> ExcludeExtensions,
+							  bool								ExcludeElfFiles,
+							  bool								ExcludeMachOFiles,
 							  uint64_t							ChunkFileSizeLimit,
 							  const ChunkedParams&				ChunkingParams)
 {
-	return std::make_unique<BasicChunkingController>(ExcludeExtensions, ChunkFileSizeLimit, ChunkingParams);
+	return std::make_unique<BasicChunkingController>(ExcludeExtensions,
+													 ExcludeElfFiles,
+													 ExcludeMachOFiles,
+													 ChunkFileSizeLimit,
+													 ChunkingParams);
 }
 std::unique_ptr<ChunkingController>
 CreateBasicChunkingController(CbObjectView Parameters)
-- 
cgit v1.2.3