aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--zenserver/cache/structuredcachestore.cpp12
-rw-r--r--zenserver/projectstore.cpp8
-rw-r--r--zenstore/basicfile.cpp64
-rw-r--r--zenstore/blockstore.cpp4
-rw-r--r--zenstore/cas.cpp4
-rw-r--r--zenstore/caslog.cpp22
-rw-r--r--zenstore/cidstore.cpp2
-rw-r--r--zenstore/compactcas.cpp624
-rw-r--r--zenstore/filecas.cpp6
-rw-r--r--zenstore/include/zenstore/basicfile.h18
-rw-r--r--zenstore/include/zenstore/blockstore.h4
-rw-r--r--zenstore/include/zenstore/caslog.h13
12 files changed, 485 insertions, 296 deletions
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 769167433..10a19f949 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -476,8 +476,8 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is
std::filesystem::path SobsPath{BucketDir / "zen.sobs"};
std::filesystem::path SlogPath{BucketDir / "zen.slog"};
- m_SobsFile.Open(SobsPath, IsNew);
- m_SlogFile.Open(SlogPath, IsNew);
+ m_SobsFile.Open(SobsPath, IsNew ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite);
+ m_SlogFile.Open(SlogPath, IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
m_SlogFile.Replay([&](const DiskIndexEntry& Entry) {
if (Entry.Key == IoHash::Zero)
@@ -905,8 +905,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
m_SlogFile.Close();
const bool IsNew = true;
- m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew);
- m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew);
+ m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite);
+ m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
m_SobsCursor = 0;
m_TotalSize = 0;
@@ -967,8 +967,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
uint64_t TmpCursor{};
std::vector<uint8_t> Chunk;
- TmpSobs.Open(TmpSobsPath, true);
- TmpLog.Open(TmpSlogPath, true);
+ TmpSobs.Open(TmpSobsPath, BasicFile::EMode::kTruncate);
+ TmpLog.Open(TmpSlogPath, CasLogFile::EMode::kTruncate);
for (const auto& Entry : ValidEntries)
{
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 58b806989..f916b2250 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -114,10 +114,10 @@ struct ProjectStore::OplogStorage : public RefCounted
CreateDirectories(m_OplogStoragePath);
}
- m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate);
+ m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
m_Oplog.Initialize();
- m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate);
+ m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite);
ZEN_ASSERT(IsPow2(m_OpsAlign));
ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1)));
@@ -653,7 +653,7 @@ ProjectStore::Project::Read()
ZEN_INFO("reading config for project '{}' from {}", Identifier, ProjectStateFilePath);
BasicFile Blob;
- Blob.Open(ProjectStateFilePath, false);
+ Blob.Open(ProjectStateFilePath, BasicFile::EMode::kRead);
IoBuffer Obj = Blob.ReadAll();
CbValidateError ValidationError = ValidateCompactBinary(MemoryView(Obj.Data(), Obj.Size()), CbValidateMode::All);
@@ -693,7 +693,7 @@ ProjectStore::Project::Write()
ZEN_INFO("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath);
BasicFile Blob;
- Blob.Open(ProjectStateFilePath, true);
+ Blob.Open(ProjectStateFilePath, BasicFile::EMode::kTruncate);
Blob.Write(Mem.Data(), Mem.Size(), 0);
Blob.Flush();
}
diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp
index 77bada95f..fc7282941 100644
--- a/zenstore/basicfile.cpp
+++ b/zenstore/basicfile.cpp
@@ -29,10 +29,10 @@ BasicFile::~BasicFile()
}
void
-BasicFile::Open(const std::filesystem::path& FileName, bool IsCreate)
+BasicFile::Open(const std::filesystem::path& FileName, EMode Mode)
{
std::error_code Ec;
- Open(FileName, IsCreate, Ec);
+ Open(FileName, Mode, Ec);
if (Ec)
{
@@ -41,16 +41,20 @@ BasicFile::Open(const std::filesystem::path& FileName, bool IsCreate)
}
void
-BasicFile::Open(const std::filesystem::path& FileName, bool IsCreate, std::error_code& Ec)
+BasicFile::Open(const std::filesystem::path& FileName, EMode Mode, std::error_code& Ec)
{
Ec.clear();
+ uint32_t ModeFlags = static_cast<uint32_t>(Mode);
#if ZEN_PLATFORM_WINDOWS
- const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING;
- DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE | DELETE;
- const DWORD dwShareMode = FILE_SHARE_READ;
- const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
- HANDLE hTemplateFile = nullptr;
+ const DWORD dwCreationDisposition = (ModeFlags & static_cast<uint32_t>(kAccessTruncate)) ? CREATE_ALWAYS : OPEN_EXISTING;
+ DWORD dwDesiredAccess = GENERIC_READ;
+ dwDesiredAccess |= (ModeFlags & static_cast<uint32_t>(kAccessWrite)) ? GENERIC_WRITE : 0;
+ dwDesiredAccess |= (ModeFlags & static_cast<uint32_t>(kAccessDelete)) ? DELETE : 0;
+
+ const DWORD dwShareMode = FILE_SHARE_READ;
+ const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+ HANDLE hTemplateFile = nullptr;
HANDLE FileHandle = CreateFile(FileName.c_str(),
dwDesiredAccess,
@@ -67,8 +71,9 @@ BasicFile::Open(const std::filesystem::path& FileName, bool IsCreate, std::error
return;
}
#else
- int OpenFlags = O_RDWR | O_CLOEXEC;
- OpenFlags |= IsCreate ? O_CREAT | O_TRUNC : 0;
+ int OpenFlags = O_CLOEXEC;
+ OpenFlags |= (ModeFlags & static_cast<uint32_t>(kAccessWrite)) ? O_RDWR : 0;
+ OpenFlags |= (ModeFlags & static_cast<uint32_t>(kAccessTruncate)) ? (O_CREAT | O_TRUNC) : 0;
int Fd = open(FileName.c_str(), OpenFlags, 0666);
if (Fd < 0)
@@ -105,27 +110,33 @@ BasicFile::Close()
void
BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
{
- const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+ const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+ uint64_t TotalFileSize = FileSize();
+ ZEN_ASSERT((FileOffset + BytesToRead) <= TotalFileSize);
while (BytesToRead)
{
const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize);
+ uint64_t NumberOfBytesRead;
#if ZEN_PLATFORM_WINDOWS
OVERLAPPED Ovl{};
Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu);
Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+ ZEN_ASSERT((FileOffset + NumberOfBytesToRead) <= TotalFileSize);
DWORD dwNumberOfBytesRead = 0;
BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl);
- ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead);
+ ZEN_ASSERT((dwNumberOfBytesRead <= NumberOfBytesToRead) && (dwNumberOfBytesRead > 0));
+ NumberOfBytesRead = dwNumberOfBytesRead;
#else
static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
- int Fd = int(uintptr_t(m_FileHandle));
- int BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
- bool Success = (BytesRead > 0);
+ int Fd = int(uintptr_t(m_FileHandle));
+ int BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
+ bool Success = (BytesRead > 0);
+ NumberOfBytesRead = static_cast<uint64_t>(BytesRead);
#endif
if (!Success)
@@ -133,9 +144,9 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle)));
}
- BytesToRead -= NumberOfBytesToRead;
- FileOffset += NumberOfBytesToRead;
- Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+ BytesToRead -= NumberOfBytesRead;
+ FileOffset += NumberOfBytesRead;
+ Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesRead;
}
}
@@ -263,7 +274,14 @@ BasicFile::FileSize()
#if ZEN_PLATFORM_WINDOWS
ULARGE_INTEGER liFileSize;
liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
-
+ if (liFileSize.LowPart == INVALID_FILE_SIZE)
+ {
+ int Error = GetLastError();
+ if (Error)
+ {
+ ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", zen::PathFromHandle(m_FileHandle)));
+ }
+ }
return uint64_t(liFileSize.QuadPart);
#else
int Fd = int(uintptr_t(m_FileHandle));
@@ -351,9 +369,7 @@ TemporaryFile::CreateTemporary(std::filesystem::path TempDirName, std::error_cod
m_TempPath = TempDirName / TempName.c_str();
- const bool IsCreate = true;
-
- Open(m_TempPath, IsCreate, Ec);
+ Open(m_TempPath, BasicFile::EMode::kTruncateDelete, Ec);
}
void
@@ -453,8 +469,8 @@ TEST_CASE("BasicFile")
ScopedCurrentDirectoryChange _;
BasicFile File1;
- CHECK_THROWS(File1.Open("zonk", false));
- CHECK_NOTHROW(File1.Open("zonk", true));
+ CHECK_THROWS(File1.Open("zonk", BasicFile::EMode::kRead));
+ CHECK_NOTHROW(File1.Open("zonk", BasicFile::EMode::kTruncate));
CHECK_NOTHROW(File1.Write("abcd", 4, 0));
CHECK(File1.FileSize() == 4);
{
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index 441b5e926..e43c3a00f 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -42,7 +42,7 @@ BlockStoreFile::InternalOpen()
{
return;
}
- m_File.Open(m_Path, false);
+ m_File.Open(m_Path, BasicFile::EMode::kDelete);
void* FileHandle = m_File.Handle();
m_IoBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, m_File.FileSize());
}
@@ -65,7 +65,7 @@ BlockStoreFile::Create(uint64_t InitialSize)
CreateDirectories(ParentPath);
}
- m_File.Open(m_Path, true);
+ m_File.Open(m_Path, BasicFile::EMode::kTruncateDelete);
if (InitialSize > 0)
{
m_File.SetFileSize(InitialSize);
diff --git a/zenstore/cas.cpp b/zenstore/cas.cpp
index b569a7dae..f30439d39 100644
--- a/zenstore/cas.cpp
+++ b/zenstore/cas.cpp
@@ -164,7 +164,7 @@ CasImpl::OpenOrCreateManifest()
std::error_code Ec;
BasicFile ManifestFile;
- ManifestFile.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec);
+ ManifestFile.Open(ManifestPath.c_str(), BasicFile::EMode::kWrite, Ec);
bool ManifestIsOk = false;
@@ -236,7 +236,7 @@ CasImpl::UpdateManifest()
ZEN_TRACE("Writing new manifest to '{}'", ManifestPath);
BasicFile Marker;
- Marker.Open(ManifestPath.c_str(), /* IsCreate */ true);
+ Marker.Open(ManifestPath.c_str(), BasicFile::EMode::kTruncate);
Marker.Write(m_ManifestObject.GetBuffer(), 0);
}
diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp
index 909605af9..906c97348 100644
--- a/zenstore/caslog.cpp
+++ b/zenstore/caslog.cpp
@@ -39,13 +39,23 @@ CasLogFile::~CasLogFile()
}
void
-CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreate)
+CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, EMode Mode)
{
m_RecordSize = RecordSize;
- std::error_code Ec;
- m_File.Open(FileName, IsCreate, Ec);
+ std::error_code Ec;
+ BasicFile::EMode FileMode = BasicFile::EMode::kRead;
+ switch (Mode)
+ {
+ case EMode::kWrite:
+ FileMode = BasicFile::EMode::kWrite;
+ break;
+ case EMode::kTruncate:
+ FileMode = BasicFile::EMode::kTruncate;
+ break;
+ }
+ m_File.Open(FileName, FileMode, Ec);
if (Ec)
{
throw std::system_error(Ec, fmt::format("Failed to open log file '{}'", FileName));
@@ -53,8 +63,12 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreat
uint64_t AppendOffset = 0;
- if (IsCreate || (m_File.FileSize() < sizeof(FileHeader)))
+ if ((Mode == EMode::kTruncate) || (m_File.FileSize() < sizeof(FileHeader)))
{
+ if (Mode == EMode::kRead)
+ {
+ throw std::runtime_error(fmt::format("Mangled log header (file to small) in '{}'", FileName));
+ }
// Initialize log by writing header
FileHeader Header = {.RecordSize = gsl::narrow<uint32_t>(RecordSize), .LogId = Oid::NewOid(), .ValidatedTail = 0};
memcpy(Header.Magic, FileHeader::MagicSequence, sizeof Header.Magic);
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 8b53a8304..5ddf32e73 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -127,7 +127,7 @@ struct CidStore::Impl
bool IsNew = !std::filesystem::exists(SlogPath);
- m_LogFile.Open(SlogPath, IsNew);
+ m_LogFile.Open(SlogPath, IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
ZEN_DEBUG("Initializing index from '{}' ({})", SlogPath, NiceBytes(m_LogFile.GetLogSize()));
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index ec0a5a7ba..9d0f72442 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -27,15 +27,15 @@ namespace zen {
struct CasDiskIndexHeader
{
- static constexpr uint32_t ExpectedMagic = 0x75696478; // 'uidx';
- static constexpr uint32_t CurrentVersion = 1;
- uint32_t Magic = ExpectedMagic;
- uint32_t Version = CurrentVersion;
- uint32_t PayloadAlignement = 0;
- uint32_t Reserved0 = 0;
- uint64_t EntryCount = 0;
- uint32_t Reserved1 = 0;
- uint32_t Reserved2 = 0;
+ static constexpr uint32_t ExpectedMagic = 0x75696478; // 'uidx';
+ static constexpr uint32_t CurrentVersion = 1;
+ uint32_t Magic = ExpectedMagic;
+ uint32_t Version = CurrentVersion;
+ uint32_t PayloadAlignment = 0;
+ uint32_t Reserved0 = 0;
+ uint64_t EntryCount = 0;
+ uint32_t Reserved1 = 0;
+ uint32_t Reserved2 = 0;
};
static_assert(sizeof(CasDiskIndexHeader) == 32);
@@ -122,10 +122,351 @@ namespace {
return RootPath / (ContainerBaseName + DataExtension);
}
+ struct LegacyCasDiskLocation
+ {
+ LegacyCasDiskLocation(uint64_t InOffset, uint64_t InSize)
+ {
+ ZEN_ASSERT(InOffset <= 0xff'ffff'ffff);
+ ZEN_ASSERT(InSize <= 0xff'ffff'ffff);
+
+ memcpy(&m_Offset[0], &InOffset, sizeof m_Offset);
+ memcpy(&m_Size[0], &InSize, sizeof m_Size);
+ }
+
+ LegacyCasDiskLocation() = default;
+
+ inline uint64_t GetOffset() const
+ {
+ uint64_t Offset = 0;
+ memcpy(&Offset, &m_Offset, sizeof m_Offset);
+ return Offset;
+ }
+
+ inline uint64_t GetSize() const
+ {
+ uint64_t Size = 0;
+ memcpy(&Size, &m_Size, sizeof m_Size);
+ return Size;
+ }
+
+ private:
+ uint8_t m_Offset[5];
+ uint8_t m_Size[5];
+ };
+
+ struct LegacyCasDiskIndexEntry
+ {
+ static const uint8_t kTombstone = 0x01;
+
+ IoHash Key;
+ LegacyCasDiskLocation Location;
+ ZenContentType ContentType = ZenContentType::kUnknownContentType;
+ uint8_t Flags = 0;
+ };
+
+ void Migrate(const std::filesystem::path& RootPath,
+ const std::string& ContainerBaseName,
+ uint64_t MaxBlockSize,
+ uint64_t PayloadAlignment,
+ bool Destructive,
+ bool Overwrite)
+ {
+ std::filesystem::path BlocksBasePath = GetBlocksBasePath(RootPath, ContainerBaseName);
+ std::filesystem::path LegacyLogPath = GetLegacyLogPath(RootPath, ContainerBaseName);
+ std::filesystem::path LegacySobsPath = GetLegacyUcasPath(RootPath, ContainerBaseName);
+ if (!std::filesystem::is_regular_file(LegacyLogPath) || !std::filesystem::is_regular_file(LegacySobsPath))
+ {
+ ZEN_DEBUG("migrate of {} SKIPPED, no legacy data found", RootPath / ContainerBaseName);
+ return;
+ }
+
+ std::filesystem::path SlogPath = GetLogPath(RootPath, ContainerBaseName);
+ if (std::filesystem::is_directory(SlogPath.parent_path()))
+ {
+ if (!Overwrite)
+ {
+ ZEN_WARN("migrate of {} SKIPPED, new content already exists", RootPath / ContainerBaseName);
+ return;
+ }
+ }
+
+ uint32_t NewBlockIndex = 0;
+ Stopwatch MigrationTimer;
+ uint64_t TotalSize = 0;
+ const auto Guard = MakeGuard([RootPath, ContainerBaseName, &MigrationTimer, &NewBlockIndex, &TotalSize] {
+ ZEN_INFO("migrated store {} to {} blocks in {} ({})",
+ RootPath / ContainerBaseName,
+ NewBlockIndex + 1,
+ NiceTimeSpanMs(MigrationTimer.GetElapsedTimeMs()),
+ NiceBytes(TotalSize));
+ });
-// void Migrate(const std::filesystem::path& RootPath, const std::string_view ContainerBaseName, uint32_t MaxBlockSize, uint64_t Alignment)
-// {
-// }
+ std::error_code Error;
+ DiskSpace Space = DiskSpaceInfo(RootPath, Error);
+ if (Error)
+ {
+ ZEN_ERROR("get disk space in {} FAILED, reason '{}'", ContainerBaseName, Error.message());
+ return;
+ }
+
+ if (Space.Free < MaxBlockSize)
+ {
+ ZEN_ERROR("legacy store migration from '{}' FAILED, required disk space {}, free {}",
+ RootPath / ContainerBaseName,
+ MaxBlockSize,
+ NiceBytes(Space.Free));
+ return;
+ }
+
+ BasicFile BlockFile;
+ BlockFile.Open(LegacySobsPath, BasicFile::EMode::kRead);
+ uint64_t FileSize = BlockFile.FileSize();
+
+ std::unordered_map<IoHash, LegacyCasDiskIndexEntry, IoHash::Hasher> LegacyDiskIndex;
+
+ TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
+ LegacyCasLog.Open(LegacyLogPath, CasLogFile::EMode::kRead);
+ LegacyCasLog.Replay([&](const LegacyCasDiskIndexEntry& Record) {
+ if (Record.Flags & LegacyCasDiskIndexEntry::kTombstone)
+ {
+ LegacyDiskIndex.erase(Record.Key);
+ return;
+ }
+ uint64_t EntryEnd = Record.Location.GetOffset() + Record.Location.GetSize();
+ if (EntryEnd > FileSize)
+ {
+ return;
+ }
+ LegacyDiskIndex[Record.Key] = Record;
+ });
+
+ uint64_t MaxUsedSize = 0;
+ for (const auto& Entry : LegacyDiskIndex)
+ {
+ const LegacyCasDiskIndexEntry& Record(Entry.second);
+ uint64_t EntryEnd = Record.Location.GetOffset() + Record.Location.GetSize();
+ if (EntryEnd > MaxUsedSize)
+ {
+ MaxUsedSize = EntryEnd;
+ }
+ TotalSize += Record.Location.GetSize();
+ }
+
+ uint64_t RequiredDiskSpace = TotalSize + ((PayloadAlignment - 1) * LegacyDiskIndex.size());
+ uint64_t MaxRequiredBlockCount = RoundUp(RequiredDiskSpace, MaxBlockSize) / MaxBlockSize;
+ if (MaxRequiredBlockCount > BlockStoreDiskLocation::MaxBlockIndex)
+ {
+ ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
+ RootPath / ContainerBaseName,
+ MaxRequiredBlockCount,
+ BlockStoreDiskLocation::MaxBlockIndex);
+ return;
+ }
+ if (Destructive)
+ {
+ if (Space.Free < (MaxBlockSize + (1 << 28)))
+ {
+ ZEN_INFO("legacy store migration from {} aborted, not enough disk space available {} ({})",
+ RootPath / ContainerBaseName,
+ NewBlockIndex + 1,
+ NiceBytes(MaxBlockSize + (1 << 28)),
+ NiceBytes(Space.Free));
+ return;
+ }
+ }
+ else
+ {
+ if (Space.Free < (RequiredDiskSpace + (1 << 28)))
+ {
+ ZEN_INFO("legacy store migration from {} aborted, not enough disk space available {} ({})",
+ RootPath / ContainerBaseName,
+ NewBlockIndex + 1,
+ NiceBytes(RequiredDiskSpace + (1 << 28)),
+ NiceBytes(Space.Free));
+ return;
+ }
+ }
+
+ CreateDirectories(SlogPath.parent_path());
+ TCasLogFile<CasDiskIndexEntry> CasLog;
+ CasLog.Open(SlogPath, CasLogFile::EMode::kTruncate);
+
+ if (Destructive && (MaxRequiredBlockCount < 2))
+ {
+ std::vector<CasDiskIndexEntry> LogEntries;
+ LogEntries.reserve(LegacyDiskIndex.size());
+
+ // We can use the block as is, just move it and add the blocks to our new log
+ for (auto& Entry : LegacyDiskIndex)
+ {
+ const LegacyCasDiskIndexEntry& Record(Entry.second);
+
+ BlockStoreLocation NewChunkLocation(0, Record.Location.GetOffset(), Record.Location.GetSize());
+ LogEntries.push_back({.Key = Entry.second.Key,
+ .Location = BlockStoreDiskLocation(NewChunkLocation, PayloadAlignment),
+ .ContentType = Record.ContentType,
+ .Flags = Record.Flags});
+ }
+ auto BlockPath = GetBlockPath(BlocksBasePath, 0);
+ CreateDirectories(BlockPath.parent_path());
+ BlockFile.Close();
+ std::filesystem::rename(LegacySobsPath, BlockPath);
+ }
+ else
+ {
+ std::vector<IoHash> ChunkHashes;
+ ChunkHashes.reserve(LegacyDiskIndex.size());
+ for (const auto& Entry : LegacyDiskIndex)
+ {
+ ChunkHashes.push_back(Entry.first);
+ }
+
+ std::sort(begin(ChunkHashes), end(ChunkHashes), [&](IoHash Lhs, IoHash Rhs) {
+ auto LhsKeyIt = LegacyDiskIndex.find(Lhs);
+ auto RhsKeyIt = LegacyDiskIndex.find(Rhs);
+ return LhsKeyIt->second.Location.GetOffset() < RhsKeyIt->second.Location.GetOffset();
+ });
+
+ uint64_t BlockSize = 0;
+ uint64_t BlockOffset = 0;
+ uint32_t BlockIndex = 0;
+ std::vector<BlockStoreLocation> NewLocations;
+ struct BlockData
+ {
+ std::vector<std::pair<IoHash, BlockStoreLocation>> Chunks;
+ uint64_t BlockOffset;
+ uint64_t BlockSize;
+ uint32_t BlockIndex;
+ };
+
+ std::vector<BlockData> BlockRanges;
+ std::vector<std::pair<IoHash, BlockStoreLocation>> Chunks;
+ BlockRanges.reserve(MaxRequiredBlockCount);
+ for (const IoHash& ChunkHash : ChunkHashes)
+ {
+ const auto& LegacyEntry = LegacyDiskIndex[ChunkHash];
+ const LegacyCasDiskLocation& LegacyChunkLocation = LegacyEntry.Location;
+
+ uint64_t ChunkOffset = LegacyChunkLocation.GetOffset();
+ uint64_t ChunkSize = LegacyChunkLocation.GetSize();
+
+ #if 0
+ {
+ std::vector<uint8_t> Data(ChunkSize);
+ BlockFile.Read(Data.data(), ChunkSize, ChunkOffset);
+ const IoHash ComputedHash = IoHash::HashBuffer(Data.data(), ChunkSize);
+ if (ComputedHash != ChunkHash)
+ {
+ ZEN_ERROR("migrating store {}, invalid hash for chunk {}. Got {}",
+ RootPath / ContainerBaseName,
+ ChunkHash,
+ ComputedHash);
+ }
+ }
+ #endif // 0
+
+ if (BlockSize == 0)
+ {
+ BlockOffset = ChunkOffset;
+ }
+ if ((BlockSize + ChunkSize) > MaxBlockSize)
+ {
+ BlockData BlockRange{.BlockOffset = BlockOffset, .BlockSize = BlockSize, .BlockIndex = BlockIndex};
+ BlockRange.Chunks.swap(Chunks);
+ BlockRanges.push_back(BlockRange);
+
+ BlockIndex++;
+ BlockOffset = ChunkOffset;
+ BlockSize = 0;
+ }
+ BlockSize = RoundUp(BlockSize, PayloadAlignment);
+ BlockStoreLocation ChunkLocation = {.BlockIndex = BlockIndex, .Offset = BlockSize, .Size = ChunkSize};
+ Chunks.push_back({ChunkHash, ChunkLocation});
+ BlockSize += ChunkSize;
+ }
+ if (BlockSize > 0)
+ {
+ BlockRanges.push_back(
+ {.Chunks = std::move(Chunks), .BlockOffset = BlockOffset, .BlockSize = BlockSize, .BlockIndex = BlockIndex});
+ }
+
+ std::reverse(BlockRanges.begin(), BlockRanges.end());
+ std::vector<std::uint8_t> Buffer(1 << 28);
+ for (size_t Idx = 0; Idx < BlockRanges.size(); ++Idx)
+ {
+ const BlockData& BlockRange = BlockRanges[Idx];
+
+ ZEN_INFO("migrating store {} {}/{} blocks, remaining {} ({})",
+ RootPath / ContainerBaseName,
+ Idx,
+ BlockRanges.size(),
+ NiceBytes(BlockRange.BlockOffset),
+ NiceBytes(TotalSize));
+
+ auto BlockPath = GetBlockPath(BlocksBasePath, BlockRange.BlockIndex);
+ BlockStoreFile ChunkBlock(BlockPath);
+ ChunkBlock.Create(BlockRange.BlockSize);
+ uint64_t Offset = 0;
+ while (Offset < BlockRange.BlockSize)
+ {
+ uint64_t Size = BlockRange.BlockSize - Offset;
+ if (Size > Buffer.size())
+ {
+ Size = Buffer.size();
+ }
+ BlockFile.Read(Buffer.data(), Size, BlockRange.BlockOffset + Offset);
+ ChunkBlock.Write(Buffer.data(), Size, Offset);
+ Offset += Size;
+ }
+ ChunkBlock.Flush();
+
+ std::vector<CasDiskIndexEntry> LogEntries;
+ LogEntries.reserve(BlockRange.Chunks.size());
+ for (const auto& Entry : BlockRange.Chunks)
+ {
+ const LegacyCasDiskIndexEntry& LegacyEntry = LegacyDiskIndex[Entry.first];
+ BlockStoreDiskLocation Location(Entry.second, PayloadAlignment);
+ LogEntries.push_back(
+ {.Key = Entry.first, .Location = Location, .ContentType = LegacyEntry.ContentType, .Flags = LegacyEntry.Flags});
+ }
+ CasLog.Append(LogEntries);
+ #if 0
+ for (const CasDiskIndexEntry& Entry : LogEntries)
+ {
+ std::vector<uint8_t> Data(Entry.Location.GetSize());
+ ChunkBlock.Read(Data.data(), Entry.Location.GetSize(), Entry.Location.GetOffset(PayloadAlignment));
+ const IoHash ComputedHash = IoHash::HashBuffer(Data.data(), Entry.Location.GetSize());
+ if (ComputedHash != Entry.Key)
+ {
+ ZEN_ERROR("migrating store {}, invalid hash for chunk {}. Got {}",
+ RootPath / ContainerBaseName,
+ Entry.Key,
+ ComputedHash);
+ }
+ }
+ #endif // 0
+
+ if (Destructive)
+ {
+ std::vector<LegacyCasDiskIndexEntry> LegacyLogEntries;
+ LegacyLogEntries.reserve(BlockRange.Chunks.size());
+ for (const auto& Entry : BlockRange.Chunks)
+ {
+ LegacyLogEntries.push_back({.Key = Entry.first, .Flags = LegacyCasDiskIndexEntry::kTombstone});
+ }
+ LegacyCasLog.Append(LegacyLogEntries);
+ BlockFile.SetFileSize(BlockRange.BlockOffset);
+ }
+ }
+ }
+ LegacyCasLog.Close();
+ CasLog.Close();
+
+ if (Destructive)
+ {
+ std::filesystem::remove(LegacyLogPath);
+ }
+ }
} // namespace
@@ -219,7 +560,7 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const
BlockStoreDiskLocation Location({.BlockIndex = WriteBlockIndex, .Offset = InsertOffset, .Size = ChunkSize}, m_PayloadAlignment);
const CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = Location};
- m_TotalSize.fetch_add(static_cast<uint64_t>(ChunkSize));
+ m_TotalSize.fetch_add(static_cast<uint64_t>(ChunkSize), std::memory_order_release);
{
RwLock::ExclusiveLockScope __(m_LocationMapLock);
m_LocationMap.emplace(ChunkHash, Location);
@@ -508,8 +849,8 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
TotalChunkCount = LocationMap.size();
std::unordered_map<uint32_t, size_t> BlockIndexToChunkMapIndex;
- std::vector<std::vector<IoHash> > KeepChunks;
- std::vector<std::vector<IoHash> > DeleteChunks;
+ std::vector<std::vector<IoHash>> KeepChunks;
+ std::vector<std::vector<IoHash>> DeleteChunks;
BlockIndexToChunkMapIndex.reserve(BlockCount);
KeepChunks.reserve(BlockCount);
@@ -774,7 +1115,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
}
BasicFile GCReserveFile;
CreateDirectories(GCReservePath.parent_path());
- GCReserveFile.Open(GCReservePath, true);
+ GCReserveFile.Open(GCReservePath, BasicFile::EMode::kTruncate);
GCReserveFile.SetFileSize(m_MaxBlockSize);
ZEN_DEBUG("recreated garbage collect reserve '{}', {} bytes", m_Config.RootDirectory / m_ContainerBaseName, NiceBytes(Space.Free));
@@ -817,7 +1158,7 @@ CasContainerStrategy::MakeIndexSnapshot()
fs::rename(SlogPath, STmplogPath);
// Open an new log
- m_CasLog.Open(SlogPath, true);
+ m_CasLog.Open(SlogPath, CasLogFile::EMode::kTruncate);
}
try
@@ -839,8 +1180,8 @@ CasContainerStrategy::MakeIndexSnapshot()
}
BasicFile ObjectIndexFile;
- ObjectIndexFile.Open(SidxPath, true);
- CasDiskIndexHeader Header = {.PayloadAlignement = gsl::narrow<uint32_t>(m_PayloadAlignment), .EntryCount = Entries.size()};
+ ObjectIndexFile.Open(SidxPath, BasicFile::EMode::kTruncate);
+ CasDiskIndexHeader Header = {.PayloadAlignment = gsl::narrow<uint32_t>(m_PayloadAlignment), .EntryCount = Entries.size()};
ObjectIndexFile.Write(&Header, sizeof(CasDiskIndexEntry), 0);
ObjectIndexFile.Write(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexEntry));
ObjectIndexFile.Close();
@@ -857,7 +1198,7 @@ CasContainerStrategy::MakeIndexSnapshot()
Records.reserve(m_LocationMap.size());
{
TCasLogFile<CasDiskIndexEntry> OldCasLog;
- OldCasLog.Open(STmplogPath, false);
+ OldCasLog.Open(STmplogPath, CasLogFile::EMode::kRead);
OldCasLog.Replay([&](const CasDiskIndexEntry& Record) { Records.push_back(Record); });
}
{
@@ -865,7 +1206,7 @@ CasContainerStrategy::MakeIndexSnapshot()
}
TCasLogFile<CasDiskIndexEntry> RecoveredCasLog;
- RecoveredCasLog.Open(SRecoveredlogPath, true);
+ RecoveredCasLog.Open(SRecoveredlogPath, CasLogFile::EMode::kWrite);
RecoveredCasLog.Append(Records);
RecoveredCasLog.Close();
@@ -896,52 +1237,6 @@ CasContainerStrategy::MakeIndexSnapshot()
}
}
-namespace {
- struct LegacyCasDiskLocation
- {
- LegacyCasDiskLocation(uint64_t InOffset, uint64_t InSize)
- {
- ZEN_ASSERT(InOffset <= 0xff'ffff'ffff);
- ZEN_ASSERT(InSize <= 0xff'ffff'ffff);
-
- memcpy(&m_Offset[0], &InOffset, sizeof m_Offset);
- memcpy(&m_Size[0], &InSize, sizeof m_Size);
- }
-
- LegacyCasDiskLocation() = default;
-
- inline uint64_t GetOffset() const
- {
- uint64_t Offset = 0;
- memcpy(&Offset, &m_Offset, sizeof m_Offset);
- return Offset;
- }
-
- inline uint64_t GetSize() const
- {
- uint64_t Size = 0;
- memcpy(&Size, &m_Size, sizeof m_Size);
- return Size;
- }
-
- private:
- uint8_t m_Offset[5];
- uint8_t m_Size[5];
- };
-
- struct LegacyCasDiskIndexEntry
- {
- static const uint8_t kTombstone = 0x01;
-
- IoHash Key;
- LegacyCasDiskLocation Location;
- ZenContentType ContentType = ZenContentType::kUnknownContentType;
- uint8_t Flags = 0;
- };
-
-} // namespace
-
-
void
CasContainerStrategy::OpenContainer(bool IsNewStore)
{
@@ -975,179 +1270,17 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
std::filesystem::remove(SidxPath);
}
CreateDirectories(SlogPath.parent_path());
- m_CasLog.Open(SlogPath, true);
+ m_CasLog.Open(SlogPath, CasLogFile::EMode::kTruncate);
}
else
{
- if (std::filesystem::is_regular_file(LegacyLogPath) && std::filesystem::is_regular_file(LegacySobsPath))
- {
- uint32_t NewBlockIndex = 0;
- Stopwatch MigrationTimer;
- uint64_t TotalSize = 0;
- const auto Guard = MakeGuard([this, &MigrationTimer, &NewBlockIndex, &TotalSize] {
- ZEN_INFO("migrated store {} to {} blocks in {} ({})",
- m_Config.RootDirectory / m_ContainerBaseName,
- NewBlockIndex + 1,
- NiceTimeSpanMs(MigrationTimer.GetElapsedTimeMs()),
- NiceBytes(TotalSize));
- });
-
- std::error_code Error;
- DiskSpace Space = DiskSpaceInfo(m_Config.RootDirectory, Error);
- if (Error)
- {
- ZEN_ERROR("get disk space in {} FAILED, reason '{}'", m_ContainerBaseName, Error.message());
- return;
- }
-
- if (Space.Free < m_MaxBlockSize) // Never let GC steal the last block space
- {
- ZEN_ERROR("legacy store migration from '{}' FAILED, required disk space {}, free {}",
- m_Config.RootDirectory / m_ContainerBaseName,
- m_MaxBlockSize,
- NiceBytes(Space.Free));
- return;
- }
-
- BasicFile BlockFile;
- BlockFile.Open(LegacySobsPath, false);
-
- std::unordered_map<IoHash, LegacyCasDiskIndexEntry, IoHash::Hasher> LegacyDiskIndex;
-
- TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
- LegacyCasLog.Open(LegacyLogPath, false);
- LegacyCasLog.Replay([&](const LegacyCasDiskIndexEntry& Record) {
- if (Record.Flags & LegacyCasDiskIndexEntry::kTombstone)
- {
- LegacyDiskIndex.erase(Record.Key);
- }
- else
- {
- LegacyDiskIndex[Record.Key] = Record;
- }
- });
-
- uint64_t MaxUsedSize = 0;
- for (const auto& Entry : LegacyDiskIndex)
- {
- const LegacyCasDiskIndexEntry& Record(Entry.second);
- uint64_t EntryEnd = Record.Location.GetOffset() + Record.Location.GetSize();
- if (EntryEnd > MaxUsedSize)
- {
- MaxUsedSize = EntryEnd;
- }
- TotalSize += Record.Location.GetSize();
- }
- LegacyCasLog.Close();
-
- BlockFile.SetFileSize(MaxUsedSize);
- uint64_t MaxRequiredBlockCount = RoundUp(MaxUsedSize, m_MaxBlockSize) / m_MaxBlockSize;
- if (MaxRequiredBlockCount > BlockStoreDiskLocation::MaxBlockIndex)
- {
- ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
- m_Config.RootDirectory / m_ContainerBaseName,
- MaxRequiredBlockCount,
- BlockStoreDiskLocation::MaxBlockIndex);
- return;
- }
-
- CreateDirectories(SlogPath.parent_path());
- m_CasLog.Open(SlogPath, true);
-
- std::vector<CasDiskIndexEntry> LogEntries;
- LogEntries.reserve(LegacyDiskIndex.size());
- if (MaxRequiredBlockCount < 2)
- {
- for (const auto& Entry : LegacyDiskIndex)
- {
- const LegacyCasDiskIndexEntry& Record(Entry.second);
- BlockStoreLocation NewChunkLocation(0, Record.Location.GetOffset(), Record.Location.GetSize());
- LogEntries.push_back({.Key = Entry.second.Key,
- .Location = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment),
- .ContentType = Record.ContentType,
- .Flags = Record.Flags});
- }
- auto BlockPath = GetBlockPath(m_BlocksBasePath, 0);
- CreateDirectories(BlockPath.parent_path());
- BlockFile.Close();
- std::filesystem::rename(LegacySobsPath, BlockPath);
- }
- else
- {
- std::vector<IoHash> ChunkHashes;
- ChunkHashes.reserve(LegacyDiskIndex.size());
- for (const auto& Entry : LegacyDiskIndex)
- {
- ChunkHashes.push_back(Entry.first);
- }
-
- // Sort from biggest position to smallest
- std::sort(begin(ChunkHashes), end(ChunkHashes), [&](IoHash Lhs, IoHash Rhs) {
- auto LhsKeyIt = LegacyDiskIndex.find(Lhs);
- auto RhsKeyIt = LegacyDiskIndex.find(Rhs);
- return RhsKeyIt->second.Location.GetOffset() < LhsKeyIt->second.Location.GetOffset();
- });
-
- std::unique_ptr<BlockStoreFile> NewBlockFile;
- uint64_t WriteOffset = 0;
-
- std::vector<uint8_t> Chunk;
- for (const IoHash& ChunkHash : ChunkHashes)
- {
- const auto& Entry = LegacyDiskIndex[ChunkHash];
- const LegacyCasDiskLocation& ChunkLocation = Entry.Location;
- Chunk.resize(ChunkLocation.GetSize());
- BlockFile.Read(Chunk.data(), Chunk.size(), ChunkLocation.GetOffset());
- if (!NewBlockFile)
- {
- auto BlockPath = GetBlockPath(m_BlocksBasePath, NewBlockIndex);
- NewBlockFile = std::make_unique<BlockStoreFile>(BlockPath);
- NewBlockFile->Create(m_MaxBlockSize);
- }
- else if (WriteOffset + Chunk.size() > m_MaxBlockSize)
- {
- m_CasLog.Append(LogEntries);
- LogEntries.clear();
- NewBlockFile.reset();
- uint64_t ChunkEnd = ChunkLocation.GetOffset() + Chunk.size();
- BlockFile.SetFileSize(ChunkEnd);
- NewBlockIndex = NewBlockIndex + 1;
- ZEN_INFO("migrating store {} {}/{} blocks, remaining {} ({})",
- m_Config.RootDirectory / m_ContainerBaseName,
- NewBlockIndex,
- MaxRequiredBlockCount,
- NiceBytes(ChunkEnd),
- NiceBytes(TotalSize));
- auto BlockPath = GetBlockPath(m_BlocksBasePath, NewBlockIndex);
- NewBlockFile = std::make_unique<BlockStoreFile>(BlockPath);
- NewBlockFile->Create(m_MaxBlockSize);
- WriteOffset = 0;
- }
- NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
- BlockStoreLocation NewChunkLocation(NewBlockIndex, WriteOffset, Chunk.size());
- LogEntries.push_back({.Key = ChunkHash,
- .Location = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment),
- .ContentType = Entry.ContentType,
- .Flags = Entry.Flags});
- WriteOffset = RoundUp(WriteOffset + Chunk.size(), m_PayloadAlignment);
- }
- NewBlockFile.reset();
- BlockFile.Close();
- }
- if (!LogEntries.empty())
- {
- m_CasLog.Append(LogEntries);
- }
- m_CasLog.Close();
-
- std::filesystem::remove(LegacyLogPath);
- CasLogEmpty = false;
- }
+ // Keep the old cache intact for now
+ Migrate(m_Config.RootDirectory, m_ContainerBaseName, m_MaxBlockSize, m_PayloadAlignment, false, true);
if (std::filesystem::is_regular_file(SidxPath))
{
BasicFile ObjectIndexFile;
- ObjectIndexFile.Open(SidxPath, false);
+ ObjectIndexFile.Open(SidxPath, BasicFile::EMode::kRead);
uint64_t Size = ObjectIndexFile.FileSize();
if (Size >= sizeof(CasDiskIndexHeader))
{
@@ -1155,7 +1288,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
CasDiskIndexHeader Header;
ObjectIndexFile.Read(&Header, sizeof(Header), 0);
if (Header.Magic == CasDiskIndexHeader::ExpectedMagic && Header.Version == CasDiskIndexHeader::CurrentVersion &&
- Header.PayloadAlignement > 0 && Header.EntryCount == ExpectedEntryCount)
+ Header.PayloadAlignment > 0 && Header.EntryCount == ExpectedEntryCount)
{
std::vector<CasDiskIndexEntry> Entries{Header.EntryCount};
ObjectIndexFile.Read(Entries.data(), Header.EntryCount * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
@@ -1164,12 +1297,12 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
{
m_LocationMap[Entry.Key] = Entry.Location;
}
- m_PayloadAlignment = Header.PayloadAlignement;
+ m_PayloadAlignment = Header.PayloadAlignment;
}
}
}
- m_CasLog.Open(SlogPath, false);
+ m_CasLog.Open(SlogPath, CasLogFile::EMode::kWrite);
m_CasLog.Replay([&](const CasDiskIndexEntry& Record) {
if (Record.Flags & CasDiskIndexEntry::kTombstone)
{
@@ -1187,7 +1320,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
for (const auto& Entry : m_LocationMap)
{
const BlockStoreDiskLocation& Location = Entry.second;
- m_TotalSize.fetch_add(Location.GetSize());
+ m_TotalSize.fetch_add(Location.GetSize(), std::memory_order_release);
BlockUsage.insert(Location.GetBlockIndex());
}
@@ -1253,7 +1386,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
BasicFile GCReserveFile;
if (std::filesystem::is_regular_file(GCReservePath))
{
- GCReserveFile.Open(GCReservePath, false);
+ GCReserveFile.Open(GCReservePath, BasicFile::EMode::kWrite);
std::uint64_t CurrentSize = GCReserveFile.FileSize();
if (CurrentSize != m_MaxBlockSize)
{
@@ -1281,7 +1414,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
if (Space.Free > m_MaxBlockSize)
{
CreateDirectories(GCReservePath.parent_path());
- GCReserveFile.Open(GCReservePath, true);
+ GCReserveFile.Open(GCReservePath, BasicFile::EMode::kTruncate);
GCReserveFile.SetFileSize(m_MaxBlockSize);
}
}
@@ -1938,7 +2071,7 @@ TEST_CASE("compactcas.legacyconversion")
if (std::filesystem::is_regular_file(SidxPath))
{
BasicFile ObjectIndexFile;
- ObjectIndexFile.Open(SidxPath, false);
+ ObjectIndexFile.Open(SidxPath, BasicFile::EMode::kRead);
uint64_t Size = ObjectIndexFile.FileSize();
if (Size >= sizeof(CasDiskIndexHeader))
{
@@ -1946,7 +2079,7 @@ TEST_CASE("compactcas.legacyconversion")
CasDiskIndexHeader Header;
ObjectIndexFile.Read(&Header, sizeof(Header), 0);
if (Header.Magic == CasDiskIndexHeader::ExpectedMagic && Header.Version == CasDiskIndexHeader::CurrentVersion &&
- Header.PayloadAlignement > 0 && Header.EntryCount == ExpectedEntryCount)
+ Header.PayloadAlignment > 0 && Header.EntryCount == ExpectedEntryCount)
{
LogEntries.resize(Header.EntryCount);
ObjectIndexFile.Read(LogEntries.data(), Header.EntryCount * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
@@ -1959,12 +2092,12 @@ TEST_CASE("compactcas.legacyconversion")
std::filesystem::path SlogPath = GetLogPath(CasConfig.RootDirectory, "test");
{
TCasLogFile<CasDiskIndexEntry> CasLog;
- CasLog.Open(SlogPath, false);
+ CasLog.Open(SlogPath, CasLogFile::EMode::kRead);
CasLog.Replay([&](const CasDiskIndexEntry& Record) { LogEntries.push_back(Record); });
}
TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
std::filesystem::path SLegacylogPath = GetLegacyLogPath(CasConfig.RootDirectory, "test");
- LegacyCasLog.Open(SLegacylogPath, true);
+ LegacyCasLog.Open(SLegacylogPath, CasLogFile::EMode::kTruncate);
for (const CasDiskIndexEntry& Entry : LogEntries)
{
BlockStoreLocation Location = Entry.Location.Get(16);
@@ -2186,7 +2319,10 @@ TEST_CASE("compactcas.threadedinsert") // * doctest::skip(true))
TEST_CASE("compactcas.migrate.large.data" * doctest::skip(true))
{
- const char* BigDataPath = "D:\\zen-data\\dc4-zen-cache-t\\cas";
+ const char* BigDataPath = "D:\\zen-data\\dc4-zen-cache-t\\cas";
+ Migrate(BigDataPath, "tobs", 1u << 28, 16, false, true);
+ Migrate(BigDataPath, "sobs", 1u << 30, 4096, false, true);
+
CasStoreConfiguration CasConfig;
CasConfig.RootDirectory = BigDataPath;
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 758c0665b..247f0806a 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -88,7 +88,7 @@ FileCasStrategy::Initialize(bool IsNewStore)
CreateDirectories(m_Config.RootDirectory);
- m_CasLog.Open(m_Config.RootDirectory / "cas.ulog", IsNewStore);
+ m_CasLog.Open(m_Config.RootDirectory / "cas.ulog", IsNewStore ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
m_CasLog.Replay([&](const FileCasIndexEntry& Entry) {
if (Entry.IsFlagSet(FileCasIndexEntry::kTombStone))
@@ -565,7 +565,7 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile&
BasicFile PayloadFile;
std::error_code Ec;
- PayloadFile.Open(Parent / File, false, Ec);
+ PayloadFile.Open(Parent / File, BasicFile::EMode::kWrite, Ec);
if (!Ec)
{
@@ -747,7 +747,7 @@ TEST_CASE("cas.file.move")
IoHash ZeroHash = IoHash::HashBuffer(ZeroBytes);
BasicFile PayloadFile;
- PayloadFile.Open(Payload1Path, true);
+ PayloadFile.Open(Payload1Path, BasicFile::EMode::kTruncate);
PayloadFile.Write(ZeroBytes, 0);
PayloadFile.Close();
diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h
index 30bb4ee8f..0be9e34f1 100644
--- a/zenstore/include/zenstore/basicfile.h
+++ b/zenstore/include/zenstore/basicfile.h
@@ -31,8 +31,21 @@ public:
BasicFile(const BasicFile&) = delete;
BasicFile& operator=(const BasicFile&) = delete;
- void Open(const std::filesystem::path& FileName, bool IsCreate);
- void Open(const std::filesystem::path& FileName, bool IsCreate, std::error_code& Ec);
+ static constexpr uint32_t kAccessTruncate = 1 << 0;
+ static constexpr uint32_t kAccessWrite = 1 << 1;
+ static constexpr uint32_t kAccessDelete = 1 << 2;
+
+ enum class EMode : uint32_t
+ {
+ kRead = 0,
+ kWrite = kAccessWrite,
+ kTruncate = kAccessWrite | kAccessTruncate,
+ kDelete = kAccessWrite | kAccessDelete,
+ kTruncateDelete = kAccessWrite | kAccessTruncate | kAccessDelete
+ };
+
+ void Open(const std::filesystem::path& FileName, EMode Mode);
+ void Open(const std::filesystem::path& FileName, EMode Mode, std::error_code& Ec);
void Close();
void Read(void* Data, uint64_t Size, uint64_t FileOffset);
void StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
@@ -58,6 +71,7 @@ public:
protected:
void* m_FileHandle = nullptr; // This is either null or valid
+private:
};
/**
diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h
index 306282665..5222ee50e 100644
--- a/zenstore/include/zenstore/blockstore.h
+++ b/zenstore/include/zenstore/blockstore.h
@@ -50,11 +50,11 @@ struct BlockStoreDiskLocation
return static_cast<std::uint32_t>(PackedOffset >> MaxOffsetBits);
}
- inline uint64_t GetOffset() const
+ inline uint64_t GetOffset(uint64_t OffsetAlignment) const
{
uint64_t PackedOffset = 0;
memcpy(&PackedOffset, &m_Offset, sizeof m_Offset);
- return PackedOffset & MaxOffset;
+ return (PackedOffset & MaxOffset) * OffsetAlignment;
}
inline uint64_t GetSize() const { return m_Size; }
diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h
index 4c1bf8196..5b6cc24a0 100644
--- a/zenstore/include/zenstore/caslog.h
+++ b/zenstore/include/zenstore/caslog.h
@@ -15,7 +15,14 @@ public:
CasLogFile();
~CasLogFile();
- void Open(std::filesystem::path FileName, size_t RecordSize, bool isCreate);
+ enum class EMode
+ {
+ kRead,
+ kWrite,
+ kTruncate
+ };
+
+ void Open(std::filesystem::path FileName, size_t RecordSize, EMode Mode);
void Append(const void* DataPointer, uint64_t DataSize);
void Replay(std::function<void(const void*)>&& Handler);
void Flush();
@@ -41,6 +48,8 @@ private:
static_assert(sizeof(FileHeader) == 64);
private:
+ void Open(std::filesystem::path FileName, size_t RecordSize, BasicFile::EMode Mode);
+
BasicFile m_File;
FileHeader m_Header;
size_t m_RecordSize = 1;
@@ -51,7 +60,7 @@ template<typename T>
class TCasLogFile : public CasLogFile
{
public:
- void Open(std::filesystem::path FileName, bool IsCreate) { CasLogFile::Open(FileName, sizeof(T), IsCreate); }
+ void Open(std::filesystem::path FileName, EMode Mode) { CasLogFile::Open(FileName, sizeof(T), Mode); }
// This should be called before the Replay() is called to do some basic sanity checking
bool Initialize() { return true; }
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-13 05:10:06 +0000