aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore/include
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenstore/include')
-rw-r--r--src/zenstore/include/zenstore/accesstime.h47
-rw-r--r--src/zenstore/include/zenstore/blockstore.h2
-rw-r--r--src/zenstore/include/zenstore/buildstore/buildstore.h186
-rw-r--r--src/zenstore/include/zenstore/cache/cachedisklayer.h1
-rw-r--r--src/zenstore/include/zenstore/cache/cacheshared.h38
-rw-r--r--src/zenstore/include/zenstore/gc.h4
6 files changed, 239 insertions, 39 deletions
diff --git a/src/zenstore/include/zenstore/accesstime.h b/src/zenstore/include/zenstore/accesstime.h
new file mode 100644
index 000000000..a28dc908b
--- /dev/null
+++ b/src/zenstore/include/zenstore/accesstime.h
@@ -0,0 +1,47 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenstore/gc.h>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch
+struct AccessTime
+{
+ explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {}
+ AccessTime& operator=(GcClock::Tick Tick) noexcept
+ {
+ SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed);
+ return *this;
+ }
+ operator GcClock::Tick() const noexcept
+ {
+ return std::chrono::duration_cast<GcClock::Duration>(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed)))
+ .count();
+ }
+
+ AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
+ AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
+ AccessTime& operator=(AccessTime&& Rhs) noexcept
+ {
+ SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
+ return *this;
+ }
+ AccessTime& operator=(const AccessTime& Rhs) noexcept
+ {
+ SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
+ return *this;
+ }
+
+private:
+ static uint32_t ToSeconds(GcClock::Tick Tick)
+ {
+ return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count());
+ }
+ std::atomic_uint32_t SecondsSinceEpoch;
+};
+
+} // namespace zen
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index 97357e5cb..0c72a13aa 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -156,7 +156,7 @@ public:
void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback);
typedef std::function<void(std::span<BlockStoreLocation> Locations)> WriteChunksCallback;
- void WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback);
+ void WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback);
IoBuffer TryGetChunk(const BlockStoreLocation& Location) const;
void Flush(bool ForceNewBlock);
diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h
new file mode 100644
index 000000000..302af5f9c
--- /dev/null
+++ b/src/zenstore/include/zenstore/buildstore/buildstore.h
@@ -0,0 +1,186 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenstore/blockstore.h>
+
+#include <zencore/iohash.h>
+#include <zenstore/accesstime.h>
+#include <zenstore/caslog.h>
+#include <zenstore/gc.h>
+#include "../compactcas.h"
+#include "../filecas.h"
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+struct BuildStoreConfig
+{
+ std::filesystem::path RootDirectory;
+ uint32_t SmallBlobBlockStoreMaxBlockSize = 256 * 1024 * 1024;
+ uint64_t SmallBlobBlockStoreMaxBlockEmbedSize = 1 * 1024 * 1024;
+ uint32_t SmallBlobBlockStoreAlignement = 16;
+ uint32_t MetadataBlockStoreMaxBlockSize = 64 * 1024 * 1024;
+ uint32_t MetadataBlockStoreAlignement = 8;
+};
+
+class BuildStore : public GcReferencer, public GcReferenceLocker //, public GcStorage
+{
+public:
+ explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc);
+ virtual ~BuildStore();
+
+ void PutBlob(const IoHash& BlobHashes, const IoBuffer& Payload);
+ IoBuffer GetBlob(const IoHash& BlobHashes);
+
+ struct BlobExistsResult
+ {
+ bool HasBody = 0;
+ bool HasMetadata = 0;
+ };
+
+ std::vector<BlobExistsResult> BlobsExists(std::span<const IoHash> BlobHashes);
+
+ void PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas);
+ std::vector<IoBuffer> GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* OptionalWorkerPool);
+
+ void Flush();
+
+private:
+ void CompactState();
+
+ uint64_t ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
+ uint64_t ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
+
+ //////// GcReferencer
+ virtual std::string GetGcName(GcCtx& Ctx) override;
+ virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
+ virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
+ virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override;
+
+ //////// GcReferenceLocker
+ virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override;
+
+#pragma pack(push)
+#pragma pack(1)
+ struct PayloadEntry
+ {
+ static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value
+ static const uint8_t kStandalone = 0x20u; // This payload is stored as a standalone value
+
+ uint8_t Flags = 0;
+ uint8_t Reserved1 = 0;
+ uint8_t Reserved2 = 0;
+ uint8_t Reserved3 = 0;
+ };
+ static_assert(sizeof(PayloadEntry) == 4);
+
+ struct PayloadDiskEntry
+ {
+ PayloadEntry Entry; // 4 bytes
+ IoHash BlobHash; // 20 bytes
+ };
+ static_assert(sizeof(PayloadDiskEntry) == 24);
+
+ struct MetadataEntry
+ {
+ BlockStoreLocation Location; // 12 bytes
+
+ ZenContentType ContentType = ZenContentType::kCOUNT; // 1 byte
+ static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value
+ uint8_t Flags = 0; // 1 byte
+
+ uint8_t Reserved1 = 0;
+ uint8_t Reserved2 = 0;
+ };
+ static_assert(sizeof(MetadataEntry) == 16);
+
+ struct MetadataDiskEntry
+ {
+ MetadataEntry Entry; // 16 bytes
+ IoHash BlobHash; // 20 bytes
+ uint8_t Reserved1 = 0;
+ uint8_t Reserved2 = 0;
+ uint8_t Reserved3 = 0;
+ uint8_t Reserved4 = 0;
+ };
+ static_assert(sizeof(MetadataDiskEntry) == 40);
+
+#pragma pack(pop)
+
+ static bool ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason);
+ static bool ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason);
+
+ struct PayloadIndex
+ {
+ uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+ operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+ PayloadIndex() = default;
+ explicit PayloadIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+ operator size_t() const { return Index; };
+ inline auto operator<=>(const PayloadIndex& Other) const = default;
+ };
+
+ struct MetadataIndex
+ {
+ uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+ operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+ MetadataIndex() = default;
+ explicit MetadataIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+ operator size_t() const { return Index; };
+ inline auto operator<=>(const MetadataIndex& Other) const = default;
+ };
+
+ struct BlobIndex
+ {
+ uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+ operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+ BlobIndex() = default;
+ explicit BlobIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+ operator size_t() const { return Index; };
+ inline auto operator<=>(const BlobIndex& Other) const = default;
+ };
+
+ struct BlobEntry
+ {
+ PayloadIndex Payload;
+ MetadataIndex Metadata;
+ AccessTime LastAccessTime;
+ };
+ static_assert(sizeof(BlobEntry) == 12);
+
+ const BuildStoreConfig m_Config;
+ GcManager& m_Gc;
+
+ RwLock m_Lock;
+
+ std::vector<PayloadEntry> m_PayloadEntries;
+ std::vector<MetadataEntry> m_MetadataEntries;
+
+ std::vector<BlobEntry> m_BlobEntries;
+ tsl::robin_map<IoHash, BlobIndex, IoHash::Hasher> m_BlobLookup;
+
+ FileCasStrategy m_LargeBlobStore;
+ CasContainerStrategy m_SmallBlobStore;
+ BlockStore m_MetadataBlockStore;
+
+ TCasLogFile<PayloadDiskEntry> m_PayloadlogFile;
+ TCasLogFile<MetadataDiskEntry> m_MetadatalogFile;
+ uint64_t m_BlobLogFlushPosition = 0;
+ uint64_t m_MetaLogFlushPosition = 0;
+
+ std::unique_ptr<HashSet> m_TrackedCacheKeys;
+
+ friend class BuildStoreGcReferenceChecker;
+ friend class BuildStoreGcReferencePruner;
+ friend class BuildStoreGcCompator;
+};
+
+void buildstore_forcelink();
+
+} // namespace zen
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 05400c784..5a51718d3 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -5,6 +5,7 @@
#include "cacheshared.h"
#include <zencore/stats.h>
+#include <zenstore/accesstime.h>
#include <zenstore/blockstore.h>
#include <zenstore/caslog.h>
diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h
index 521c78bb1..ef1b803de 100644
--- a/src/zenstore/include/zenstore/cache/cacheshared.h
+++ b/src/zenstore/include/zenstore/cache/cacheshared.h
@@ -72,42 +72,4 @@ struct CacheContentStats
bool IsKnownBadBucketName(std::string_view BucketName);
bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer);
-//////////////////////////////////////////////////////////////////////////
-
-// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch
-struct AccessTime
-{
- explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {}
- AccessTime& operator=(GcClock::Tick Tick) noexcept
- {
- SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed);
- return *this;
- }
- operator GcClock::Tick() const noexcept
- {
- return std::chrono::duration_cast<GcClock::Duration>(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed)))
- .count();
- }
-
- AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
- AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
- AccessTime& operator=(AccessTime&& Rhs) noexcept
- {
- SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
- return *this;
- }
- AccessTime& operator=(const AccessTime& Rhs) noexcept
- {
- SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
- return *this;
- }
-
-private:
- static uint32_t ToSeconds(GcClock::Tick Tick)
- {
- return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count());
- }
- std::atomic_uint32_t SecondsSinceEpoch;
-};
-
} // namespace zen
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 3daae0a93..67aadef71 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -55,6 +55,7 @@ struct GcSettings
{
GcClock::TimePoint CacheExpireTime = GcClock::Now();
GcClock::TimePoint ProjectStoreExpireTime = GcClock::Now();
+ GcClock::TimePoint BuildStoreExpireTime = GcClock::Now();
bool CollectSmallObjects = false;
bool IsDeleteMode = false;
bool SkipCidDelete = false;
@@ -412,6 +413,7 @@ struct GcSchedulerConfig
std::chrono::seconds Interval{};
std::chrono::seconds MaxCacheDuration{86400};
std::chrono::seconds MaxProjectStoreDuration{604800};
+ std::chrono::seconds MaxBuildStoreDuration{604800};
bool CollectSmallObjects = true;
bool Enabled = true;
uint64_t DiskReserveSize = 1ul << 28;
@@ -496,6 +498,7 @@ public:
bool CollectSmallObjects = false;
std::chrono::seconds MaxCacheDuration = std::chrono::seconds::max();
std::chrono::seconds MaxProjectStoreDuration = std::chrono::seconds::max();
+ std::chrono::seconds MaxBuildStoreDuration = std::chrono::seconds::max();
uint64_t DiskSizeSoftLimit = 0;
bool SkipCid = false;
bool SkipDelete = false;
@@ -528,6 +531,7 @@ private:
void SchedulerThread();
bool CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
const GcClock::TimePoint& ProjectStoreExpireTime,
+ const GcClock::TimePoint& BuildStoreExpireTime,
bool Delete,
bool CollectSmallObjects,
bool SkipCid,
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-31 02:10:11 +0000