optional compress of block chunks (#326)

- Feature: zenserver: Add command line option `--gc-buildstore-duration-seconds` to control GC life time of build store data
 - Improvement: ELF and MachO executable files are no longer chunked
- Improvement: Compress chunks in blocks that encloses a full file (such as small executables)
- Bugfix: Strip path delimiter at end of string in StringToPath

1 files changed, 26 insertions, 5 deletions

diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp
index 8674aab75..eb36be049 100644
--- a/src/zenstore/buildstore/buildstore.cpp
+++ b/src/zenstore/buildstore/buildstore.cpp
@@ -340,7 +340,7 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O
 					ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex);
 					ResultContentTypes[Index] = ExistingMetadataEntry.ContentType;
 				}
-				ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::Tick());
+				ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::TickCount());
 			}
 		}
 	}
@@ -543,7 +543,17 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys
 			if (Record.Entry.Flags & PayloadEntry::kTombStone)
 			{
 				// Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation
-				m_BlobLookup.erase(Record.BlobHash);
+				if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end())
+				{
+					if (!m_BlobEntries[ExistingIt->second].Metadata)
+					{
+						m_BlobLookup.erase(ExistingIt);
+					}
+					else
+					{
+						m_BlobEntries[ExistingIt->second].Payload = {};
+					}
+				}
 				return;
 			}
 
@@ -575,7 +585,7 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys
 				m_PayloadEntries.push_back(Record.Entry);
 
 				const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size()));
-				m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::Tick())});
+				m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
 				m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex);
 			}
 		},
@@ -635,7 +645,18 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy
 			if (Record.Entry.Flags & MetadataEntry::kTombStone)
 			{
 				// Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation
-				m_BlobLookup.erase(Record.BlobHash);
+				// Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation
+				if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end())
+				{
+					if (!m_BlobEntries[ExistingIt->second].Payload)
+					{
+						m_BlobLookup.erase(ExistingIt);
+					}
+					else
+					{
+						m_BlobEntries[ExistingIt->second].Metadata = {};
+					}
+				}
 				return;
 			}
 
@@ -667,7 +688,7 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy
 				m_MetadataEntries.push_back(Record.Entry);
 
 				const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size()));
-				m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::Tick())});
+				m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())});
 				m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex);
 			}
 		},