build store save access times (#341)v5.6.3-pre0

* save payload size in log for buildstore
* read/write access times and manifest for buldstore
* use retry when removing temporary files

2 files changed, 40 insertions, 11 deletions

diff --git a/src/zenstore/include/zenstore/accesstime.h b/src/zenstore/include/zenstore/accesstime.h
index a28dc908b..e53937b52 100644
--- a/src/zenstore/include/zenstore/accesstime.h
+++ b/src/zenstore/include/zenstore/accesstime.h
@@ -11,10 +11,10 @@ namespace zen {
 // This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch
 struct AccessTime
 {
-	explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {}
+	explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSecondsSinceEpoch(Tick)) {}
 	AccessTime& operator=(GcClock::Tick Tick) noexcept
 	{
-		SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed);
+		SecondsSinceEpoch.store(ToSecondsSinceEpoch(Tick), std::memory_order_relaxed);
 		return *this;
 	}
 	operator GcClock::Tick() const noexcept
@@ -36,8 +36,14 @@ struct AccessTime
 		return *this;
 	}
 
+	void SetSecondsSinceEpoch(uint32_t InSecondsSinceEpoch) { SecondsSinceEpoch.store(InSecondsSinceEpoch); }
+
+	uint32_t GetSecondsSinceEpoch() const { return SecondsSinceEpoch.load(); }
+
 private:
-	static uint32_t ToSeconds(GcClock::Tick Tick)
+	AccessTime(uint32_t InSecondsSinceEpoch) noexcept : SecondsSinceEpoch(InSecondsSinceEpoch) {}
+
+	static uint32_t ToSecondsSinceEpoch(GcClock::Tick Tick)
 	{
 		return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count());
 	}
diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h
index 302af5f9c..d88e682de 100644
--- a/src/zenstore/include/zenstore/buildstore/buildstore.h
+++ b/src/zenstore/include/zenstore/buildstore/buildstore.h
@@ -48,11 +48,19 @@ public:
 
 	void Flush();
 
+#if ZEN_WITH_TESTS
+	std::optional<AccessTime> GetLastAccessTime(const IoHash& Key) const;
+	bool					  SetLastAccessTime(const IoHash& Key, const AccessTime& Time);
+#endif	// ZEN_WITH_TESTS
 private:
+	LoggerRef Log() { return m_Log; }
+
 	void CompactState();
 
 	uint64_t ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
 	uint64_t ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
+	void	 WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath);
+	void	 ReadAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath);
 
 	//////// GcReferencer
 	virtual std::string						   GetGcName(GcCtx& Ctx) override;
@@ -67,22 +75,35 @@ private:
 #pragma pack(1)
 	struct PayloadEntry
 	{
+		PayloadEntry() {}
+		PayloadEntry(uint64_t Flags, uint64_t Size)
+		{
+			ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size);
+			ZEN_ASSERT((Flags & (kTombStone | kStandalone)) == Flags);
+			FlagsAndSize = (Size << 8) | Flags;
+		}
 		static const uint8_t kTombStone	 = 0x10u;  // Represents a deleted key/value
 		static const uint8_t kStandalone = 0x20u;  // This payload is stored as a standalone value
 
-		uint8_t Flags	  = 0;
-		uint8_t Reserved1 = 0;
-		uint8_t Reserved2 = 0;
-		uint8_t Reserved3 = 0;
+		uint64_t FlagsAndSize = 0;
+		uint64_t GetSize() const { return FlagsAndSize >> 8; }
+		uint8_t	 GetFlags() const { return uint8_t(FlagsAndSize & 0xff); }
+		void	 AddFlag(uint8_t Flag) { FlagsAndSize |= Flag; }
+		void	 SetSize(uint64_t Size)
+		{
+			ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size);
+			FlagsAndSize = (Size << 8) | (FlagsAndSize & 0xff);
+		}
+		void SetFlags(uint8_t Flags) { FlagsAndSize = (FlagsAndSize & 0xffffffffffffff00u) | Flags; }
 	};
-	static_assert(sizeof(PayloadEntry) == 4);
+	static_assert(sizeof(PayloadEntry) == 8);
 
 	struct PayloadDiskEntry
 	{
-		PayloadEntry Entry;		// 4 bytes
+		PayloadEntry Entry;		// 8 bytes
 		IoHash		 BlobHash;	// 20 bytes
 	};
-	static_assert(sizeof(PayloadDiskEntry) == 24);
+	static_assert(sizeof(PayloadDiskEntry) == 28);
 
 	struct MetadataEntry
 	{
@@ -154,10 +175,11 @@ private:
 	};
 	static_assert(sizeof(BlobEntry) == 12);
 
+	LoggerRef			   m_Log;
 	const BuildStoreConfig m_Config;
 	GcManager&			   m_Gc;
 
-	RwLock m_Lock;
+	mutable RwLock m_Lock;
 
 	std::vector<PayloadEntry>  m_PayloadEntries;
 	std::vector<MetadataEntry> m_MetadataEntries;
@@ -175,6 +197,7 @@ private:
 	uint64_t					   m_MetaLogFlushPosition = 0;
 
 	std::unique_ptr<HashSet> m_TrackedCacheKeys;
+	std::atomic<uint64_t>	 m_LastAccessTimeUpdateCount;
 
 	friend class BuildStoreGcReferenceChecker;
 	friend class BuildStoreGcReferencePruner;