aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/cache/cachedisklayer.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/cache/cachedisklayer.cpp')
-rw-r--r--src/zenserver/cache/cachedisklayer.cpp113
1 files changed, 75 insertions, 38 deletions
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 955ab3a04..2c344dd1d 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -255,8 +255,8 @@ public:
uint64_t GetSidecarSize() const { return m_ManifestEntryCount * sizeof(ManifestData); }
void WriteSidecarFile(const std::filesystem::path& SidecarPath,
uint64_t SnapshotLogPosition,
- ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
- std::vector<AccessTime>&& AccessTimes,
+ const ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
+ const std::vector<AccessTime>& AccessTimes,
const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads,
const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas);
bool ReadSidecarFile(ZenCacheDiskLayer::CacheBucket& Bucket,
@@ -265,11 +265,11 @@ public:
std::vector<AccessTime>& AccessTimes,
std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads);
- IoBuffer MakeManifest(const Oid& BucketId,
- ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
- std::vector<AccessTime>&& AccessTimes,
- const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads,
- const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas);
+ IoBuffer MakeManifest(const Oid& BucketId,
+ ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
+ std::vector<AccessTime>&& AccessTimes,
+ std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads,
+ std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas);
CbObject Manifest;
@@ -399,11 +399,11 @@ BucketManifestSerializer::GenerateNewManifest(std::filesystem::path ManifestPath
}
IoBuffer
-BucketManifestSerializer::MakeManifest(const Oid& BucketId,
- ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
- std::vector<AccessTime>&& AccessTimes,
- const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads,
- const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas)
+BucketManifestSerializer::MakeManifest(const Oid& BucketId,
+ ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
+ std::vector<AccessTime>&& AccessTimes,
+ std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads,
+ std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas)
{
using namespace std::literals;
@@ -579,8 +579,8 @@ BucketManifestSerializer::ReadSidecarFile(ZenCacheDiskLayer::CacheBucket&
void
BucketManifestSerializer::WriteSidecarFile(const std::filesystem::path& SidecarPath,
uint64_t SnapshotLogPosition,
- ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
- std::vector<AccessTime>&& AccessTimes,
+ const ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
+ const std::vector<AccessTime>& AccessTimes,
const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads,
const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas)
{
@@ -605,8 +605,11 @@ BucketManifestSerializer::WriteSidecarFile(const std::filesystem::path&
{
uint64_t WriteOffset = sizeof Header;
- BasicFileWriter SidecarWriter(SidecarFile, 128 * 1024);
+ // BasicFileWriter SidecarWriter(SidecarFile, 128 * 1024);
+ std::vector<ManifestData> ManifestDataBuffer;
+ const size_t MaxManifestDataBufferCount = Min(Index.size(), 4096u); // 256 Kb
+ ManifestDataBuffer.reserve(MaxManifestDataBufferCount);
for (auto& Kv : Index)
{
const IoHash& Key = Kv.first;
@@ -621,12 +624,24 @@ BucketManifestSerializer::WriteSidecarFile(const std::filesystem::path&
RawSize = MetaDatas[MetaIndex].RawSize;
}
- ManifestData ManifestEntry =
- {.Key = Key, .Timestamp = AccessTimes[PlIndex], .RawHash = RawHash, .Padding_0 = 0, .RawSize = RawSize, .Padding_1 = 0};
-
- SidecarWriter.Write(&ManifestEntry, sizeof ManifestEntry, WriteOffset);
-
- WriteOffset += sizeof ManifestEntry;
+ ManifestDataBuffer.emplace_back(ManifestData{.Key = Key,
+ .Timestamp = AccessTimes[PlIndex],
+ .RawHash = RawHash,
+ .Padding_0 = 0,
+ .RawSize = RawSize,
+ .Padding_1 = 0});
+ if (ManifestDataBuffer.size() == MaxManifestDataBufferCount)
+ {
+ const uint64_t WriteSize = sizeof(ManifestData) * ManifestDataBuffer.size();
+ SidecarFile.Write(ManifestDataBuffer.data(), WriteSize, WriteOffset);
+ WriteOffset += WriteSize;
+ ManifestDataBuffer.clear();
+ ManifestDataBuffer.reserve(MaxManifestDataBufferCount);
+ }
+ }
+ if (ManifestDataBuffer.size() > 0)
+ {
+ SidecarFile.Write(ManifestDataBuffer.data(), sizeof(ManifestData) * ManifestDataBuffer.size(), WriteOffset);
}
}
@@ -1380,12 +1395,12 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
{
try
{
- bool UseLegacyScheme = false;
- uint64_t SidecarSize = 0;
+ bool UseLegacyScheme = false;
IoBuffer Buffer;
BucketManifestSerializer ManifestWriter;
+ if (UseLegacyScheme)
{
std::vector<AccessTime> AccessTimes;
std::vector<BucketPayload> Payloads;
@@ -1403,16 +1418,41 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
MetaDatas = m_MetaDatas;
}
- if (UseLegacyScheme)
+ Buffer = ManifestWriter.MakeManifest(m_BucketId,
+ std::move(Index),
+ std::move(AccessTimes),
+ std::move(Payloads),
+ std::move(MetaDatas));
+ const uint64_t RequiredSpace = Buffer.GetSize() + 1024 * 512;
+
+ std::error_code Error;
+ DiskSpace Space = DiskSpaceInfo(m_BucketDir, Error);
+ if (Error)
{
- Buffer = ManifestWriter.MakeManifest(m_BucketId, std::move(Index), std::move(AccessTimes), Payloads, MetaDatas);
+ ZEN_WARN("get disk space in '{}' FAILED, reason: '{}'", m_BucketDir, Error.message());
+ return;
}
- else
+ bool EnoughSpace = Space.Free >= RequiredSpace;
+ if (!EnoughSpace)
{
- const uint64_t EntryCount = Index.size();
- Buffer = ManifestWriter.MakeSidecarManifest(m_BucketId, EntryCount);
- SidecarSize = ManifestWriter.GetSidecarSize();
+ uint64_t ReclaimedSpace = ClaimDiskReserveFunc();
+ EnoughSpace = (Space.Free + ReclaimedSpace) >= RequiredSpace;
}
+ if (!EnoughSpace)
+ {
+ ZEN_WARN("not enough free disk space in '{}'. FAILED to save manifest of size {}",
+ m_BucketDir,
+ NiceBytes(Buffer.GetSize()));
+ return;
+ }
+ }
+ else
+ {
+ RwLock::SharedLockScope IndexLock(m_IndexLock);
+ WriteIndexSnapshot();
+ const uint64_t EntryCount = m_Index.size();
+ Buffer = ManifestWriter.MakeSidecarManifest(m_BucketId, EntryCount);
+ uint64_t SidecarSize = ManifestWriter.GetSidecarSize();
const uint64_t RequiredSpace = SidecarSize + Buffer.GetSize() + 1024 * 512;
@@ -1437,15 +1477,12 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
return;
}
- if (!UseLegacyScheme)
- {
- ManifestWriter.WriteSidecarFile(GetMetaPath(m_BucketDir, m_BucketName),
- m_LogFlushPosition,
- std::move(Index),
- std::move(AccessTimes),
- Payloads,
- MetaDatas);
- }
+ ManifestWriter.WriteSidecarFile(GetMetaPath(m_BucketDir, m_BucketName),
+ m_LogFlushPosition,
+ m_Index,
+ m_AccessTimes,
+ m_Payloads,
+ m_MetaDatas);
}
std::filesystem::path ManifestPath = GetManifestPath(m_BucketDir, m_BucketName);
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-12 00:53:28 +0000