diff options
Diffstat (limited to 'src/zenserver/projectstore/projectstore.cpp')
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 1961 |
1 files changed, 1760 insertions, 201 deletions
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 415c80078..70045c13c 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -19,6 +19,7 @@ #include <zenstore/cidstore.h> #include <zenstore/scrubcontext.h> #include <zenutil/cache/rpcrecording.h> +#include <zenutil/openprocesscache.h> #include <zenutil/packageformat.h> #include <zenutil/referencemetadata.h> #include <zenutil/workerpools.h> @@ -314,6 +315,27 @@ namespace { static_assert(sizeof(OplogIndexHeader) == 64); + static std::uint64_t GetModificationTagFromRawHash(const IoHash& Hash) + { + IoHash::Hasher H; + return H(Hash); + } + + static std::uint64_t GetModificationTagFromModificationTime(IoBuffer FileBuffer) + { + IoBufferFileReference FileRef; + if (FileBuffer.GetFileReference(FileRef)) + { + std::error_code Ec; + uint64_t ModificationTick = GetModificationTickFromHandle(FileRef.FileHandle, Ec); + if (!Ec) + { + return ModificationTick; + } + } + return {}; + } + } // namespace ////////////////////////////////////////////////////////////////////////// @@ -1920,19 +1942,27 @@ ProjectStore::Oplog::GetChunkByRawHash(const IoHash& RawHash) } bool -ProjectStore::Oplog::IterateChunks(std::span<IoHash> RawHashes, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit) -{ - return m_CidStore.IterateChunks(RawHashes, AsyncCallback, OptionalWorkerPool, LargeSizeLimit); +ProjectStore::Oplog::IterateChunks(std::span<IoHash> RawHashes, + bool IncludeModTag, + const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) +{ + return m_CidStore.IterateChunks( + RawHashes, + [&](size_t Index, const IoBuffer& Payload) { + return AsyncCallback(Index, Payload, IncludeModTag ? GetModificationTagFromRawHash(RawHashes[Index]) : 0); + }, + OptionalWorkerPool, + LargeSizeLimit); } bool -ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit) +ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, + bool IncludeModTag, + const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) { std::vector<size_t> CidChunkIndexes; std::vector<IoHash> CidChunkHashes; @@ -1960,12 +1990,6 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } } } - m_CidStore.IterateChunks( - CidChunkHashes, - [&](size_t Index, const IoBuffer& Payload) { return AsyncCallback(CidChunkIndexes[Index], Payload); }, - OptionalWorkerPool, - LargeSizeLimit); - if (OptionalWorkerPool) { std::atomic_bool Result = true; @@ -1979,7 +2003,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } WorkLatch.AddCount(1); OptionalWorkerPool->ScheduleWork( - [this, &WorkLatch, &ChunkIds, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() { + [this, &WorkLatch, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() { auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); if (Result.load() == false) { @@ -1995,7 +2019,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[ChunkIndex], FilePath); } - if (!AsyncCallback(FileChunkIndex, Payload)) + if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0)) { Result.store(false); } @@ -2012,6 +2036,18 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, }); } + if (!CidChunkHashes.empty()) + { + m_CidStore.IterateChunks( + CidChunkHashes, + [&](size_t Index, const IoBuffer& Payload) { + size_t CidChunkIndex = CidChunkIndexes[Index]; + return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0); + }, + OptionalWorkerPool, + LargeSizeLimit); + } + WorkLatch.CountDown(); WorkLatch.Wait(); @@ -2019,6 +2055,18 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } else { + if (!CidChunkHashes.empty()) + { + m_CidStore.IterateChunks( + CidChunkHashes, + [&](size_t Index, const IoBuffer& Payload) { + size_t CidChunkIndex = CidChunkIndexes[Index]; + return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0); + }, + OptionalWorkerPool, + LargeSizeLimit); + } + for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++) { size_t FileChunkIndex = FileChunkIndexes[ChunkIndex]; @@ -2026,7 +2074,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath); if (Payload) { - bool Result = AsyncCallback(FileChunkIndex, Payload); + bool Result = AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0); if (!Result) { return false; @@ -2038,7 +2086,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } IoBuffer -ProjectStore::Oplog::FindChunk(const Oid& ChunkId) +ProjectStore::Oplog::FindChunk(const Oid& ChunkId, uint64_t* OptOutModificationTag) { RwLock::SharedLockScope OplogLock(m_OplogLock); if (!m_Storage) @@ -2051,7 +2099,12 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId) IoHash ChunkHash = ChunkIt->second; OplogLock.ReleaseNow(); - return m_CidStore.FindChunkByCid(ChunkHash); + IoBuffer Result = m_CidStore.FindChunkByCid(ChunkHash); + if (Result && OptOutModificationTag != nullptr) + { + *OptOutModificationTag = GetModificationTagFromRawHash(ChunkHash); + } + return Result; } if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end()) @@ -2065,6 +2118,10 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId) { ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkId, FilePath); } + else if (OptOutModificationTag != nullptr) + { + *OptOutModificationTag = GetModificationTagFromModificationTime(Result); + } return Result; } @@ -2073,7 +2130,12 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId) IoHash ChunkHash = MetaIt->second; OplogLock.ReleaseNow(); - return m_CidStore.FindChunkByCid(ChunkHash); + IoBuffer Result = m_CidStore.FindChunkByCid(ChunkHash); + if (Result && OptOutModificationTag != nullptr) + { + *OptOutModificationTag = GetModificationTagFromRawHash(ChunkHash); + } + return Result; } return {}; @@ -2109,7 +2171,7 @@ ProjectStore::Oplog::GetAllChunksInfo() for (ChunkInfo& Info : InfoArray) { - if (IoBuffer Chunk = FindChunk(Info.ChunkId)) + if (IoBuffer Chunk = FindChunk(Info.ChunkId, nullptr)) { Info.ChunkSize = Chunk.GetSize(); } @@ -3032,7 +3094,7 @@ ProjectStore::Project::WriteAccessTimes() { using namespace std::literals; - CbObjectWriter Writer; + CbObjectWriter Writer(32 + (m_LastAccessTimes.size() * 16)); { RwLock::SharedLockScope _(m_LastAccessTimesLock); @@ -3557,11 +3619,17 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const ////////////////////////////////////////////////////////////////////////// -ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, const Configuration& Config) +ProjectStore::ProjectStore(CidStore& Store, + std::filesystem::path BasePath, + GcManager& Gc, + JobQueue& JobQueue, + OpenProcessCache& InOpenProcessCache, + const Configuration& Config) : m_Log(logging::Get("project")) , m_Gc(Gc) , m_CidStore(Store) , m_JobQueue(JobQueue) +, m_OpenProcessCache(InOpenProcessCache) , m_ProjectBasePath(BasePath) , m_Config(Config) , m_DiskWriteBlocker(Gc.GetDiskWriteBlocker()) @@ -3972,7 +4040,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, FoundLog->IterateChunks( Ids, - [&](size_t Index, const IoBuffer& Payload) { + false, + [&](size_t Index, const IoBuffer& Payload, uint64_t /*ModTag*/) { try { if (Payload) @@ -4033,7 +4102,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, 256u * 1024u); } - CbObjectWriter Response; + CbObjectWriter Response(64u + Count * ((WantsIdField ? (5 + sizeof(Oid::OidBits)) : 0) + (WantsServerPathField ? 64 : 0) + + (WantsClientPathField ? 64 : 0) + (WantsSizeField ? 16 : 0) + (WantsRawSizeField ? 16 : 0))); Response.BeginArray("files"sv); for (size_t Index = 0; Index < Count; Index++) { @@ -4102,7 +4172,16 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, std::vector<uint64_t> RawSizes; std::vector<uint64_t> Sizes; - size_t Count = 0; + size_t Count = 0; + size_t EstimatedCount = FoundLog->OplogCount(); + if (WantsIdField) + { + Ids.reserve(EstimatedCount); + } + if (WantsRawHashField || WantsRawSizeField || WantsSizeField) + { + Hashes.reserve(EstimatedCount); + } FoundLog->IterateChunkMap([&](const Oid& Id, const IoHash& Hash) { if (WantsIdField) { @@ -4129,7 +4208,8 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); (void)FoundLog->IterateChunks( Hashes, - [&](size_t Index, const IoBuffer& Payload) -> bool { + false, + [&](size_t Index, const IoBuffer& Payload, uint64_t /*ModTag*/) -> bool { try { if (Payload) @@ -4190,7 +4270,9 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, 256u * 1024u); } - CbObjectWriter Response; + CbObjectWriter Response(64u + Count * ((WantsIdField ? (5 + sizeof(Oid::OidBits)) : 0) + + (WantsRawHashField ? (10 + sizeof(IoHash::Hash)) : 0) + (WantsSizeField ? 16 : 0) + + (WantsRawSizeField ? 16 : 0))); Response.BeginArray("chunkinfos"sv); for (size_t Index = 0; Index < Count; Index++) @@ -4250,7 +4332,7 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId, const Oid Obj = Oid::FromHexString(ChunkId); - IoBuffer Chunk = FoundLog->FindChunk(Obj); + IoBuffer Chunk = FoundLog->FindChunk(Obj, nullptr); if (!Chunk) { return {HttpResponseCode::NotFound, {}}; @@ -4284,7 +4366,8 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, uint64_t Size, ZenContentType AcceptType, CompositeBuffer& OutChunk, - ZenContentType& OutContentType) + ZenContentType& OutContentType, + uint64_t* OptionalInOutModificationTag) { if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) { @@ -4293,41 +4376,19 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, const Oid Obj = Oid::FromHexString(ChunkId); - return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType); + return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType, OptionalInOutModificationTag); } -std::pair<HttpResponseCode, std::string> -ProjectStore::GetChunkRange(const std::string_view ProjectId, - const std::string_view OplogId, - Oid ChunkId, - uint64_t Offset, - uint64_t Size, - ZenContentType AcceptType, - CompositeBuffer& OutChunk, - ZenContentType& OutContentType) +static std::pair<HttpResponseCode, std::string> +ExtractRange(IoBuffer&& Chunk, + uint64_t Offset, + uint64_t Size, + ZenContentType AcceptType, + ZenContentType& OutContentType, + CompositeBuffer& OutChunk, + IoHash& OutRawHash, + uint64_t& OutRawSize) { - bool IsOffset = Offset != 0 || Size != ~(0ull); - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); - if (!FoundLog) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - IoBuffer Chunk = FoundLog->FindChunk(ChunkId); - if (!Chunk) - { - return {HttpResponseCode::NotFound, {}}; - } - OutContentType = Chunk.GetContentType(); if (OutContentType == ZenContentType::kCompressedBinary) @@ -4335,26 +4396,47 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, IoHash RawHash; uint64_t RawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), RawHash, RawSize); - ZEN_ASSERT(!Compressed.IsNull()); + if (!Compressed) + { + return {HttpResponseCode::InternalServerError, "malformed compressed buffer"}; + } + + const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == RawSize)); - if (IsOffset) + if (IsFullRange) + { + if (AcceptType == ZenContentType::kBinary) + { + OutChunk = Compressed.DecompressToComposite(); + OutContentType = ZenContentType::kBinary; + } + else + { + OutChunk = Compressed.GetCompressed(); + } + OutRawSize = 0; + } + else { if (Size == ~(0ull) || (Offset + Size) > RawSize) { - Size = RawSize - Offset; + if (Offset < RawSize) + { + Size = RawSize - Offset; + } + else + { + Size = 0; + } } if (Size == 0) { - return { - HttpResponseCode::NotFound, - fmt::format("Chunk request for range outside of chunk '{}/{}'. Request: Chunk: {}, Offset: {}, Size: {}, ChunkSize: {}", - ProjectId, - OplogId, - ChunkId, - Offset, - Size, - RawSize)}; + return {HttpResponseCode::NotFound, + fmt::format("Chunk request for range outside of compressed chunk. Offset: {}, Size: {}, ChunkSize: {}", + Offset, + Size, + RawSize)}; } if (AcceptType == ZenContentType::kBinary) @@ -4368,47 +4450,93 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, // The client will have to compensate for any offsets that do not land on an even block size multiple OutChunk = Compressed.GetRange(Offset, Size).GetCompressed(); } + OutRawSize = RawSize; + } + OutRawHash = RawHash; + } + else + { + const uint64_t ChunkSize = Chunk.GetSize(); + + const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == ChunkSize)); + if (IsFullRange) + { + OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk))); + OutRawSize = 0; } else { - if (AcceptType == ZenContentType::kBinary) + if (Size == ~(0ull) || (Offset + Size) > ChunkSize) { - OutChunk = Compressed.DecompressToComposite(); - OutContentType = ZenContentType::kBinary; + if (Offset < ChunkSize) + { + Size = ChunkSize - Offset; + } + else + { + Size = 0; + } } - else + + if (Size == 0) { - OutChunk = Compressed.GetCompressed(); + return {HttpResponseCode::NotFound, + fmt::format("Chunk request for range outside of chunk. Offset: {}, Size: {}, ChunkSize: {}", Offset, Size, Size)}; } + + OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size))); + OutRawSize = ChunkSize; } } - else if (IsOffset) + return {HttpResponseCode::OK, {}}; +} + +std::pair<HttpResponseCode, std::string> +ProjectStore::GetChunkRange(const std::string_view ProjectId, + const std::string_view OplogId, + Oid ChunkId, + uint64_t Offset, + uint64_t Size, + ZenContentType AcceptType, + CompositeBuffer& OutChunk, + ZenContentType& OutContentType, + uint64_t* OptionalInOutModificationTag) +{ + Ref<ProjectStore::Project> Project = OpenProject(ProjectId); + if (!Project) { - if (Size == ~(0ull) || (Offset + Size) > Chunk.GetSize()) - { - Size = Chunk.GetSize() - Offset; - } + return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)}; + } + Project->TouchProject(); - if (Size == 0) - { - return {HttpResponseCode::NotFound, - fmt::format("Chunk request for range outside of chunk '{}/{}'. Request: Chunk: {}, Offset: {}, Size: {}, ChunkSize: {}", - ProjectId, - OplogId, - ChunkId, - Offset, - Size, - Chunk.GetSize())}; - } + ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)}; + } + Project->TouchOplog(OplogId); - OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size))); + uint64_t OldTag = OptionalInOutModificationTag == nullptr ? 0 : *OptionalInOutModificationTag; + IoBuffer Chunk = FoundLog->FindChunk(ChunkId, OptionalInOutModificationTag); + if (!Chunk) + { + return {HttpResponseCode::NotFound, {}}; } - else + if (OptionalInOutModificationTag != nullptr && OldTag == *OptionalInOutModificationTag) { - OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk))); + return {HttpResponseCode::NotModified, {}}; } - return {HttpResponseCode::OK, {}}; + IoHash _; + uint64_t __; + std::pair<HttpResponseCode, std::string> Result = + ExtractRange(std::move(Chunk), Offset, Size, AcceptType, OutContentType, OutChunk, /*OutRawHash*/ _, /*OutRawSize*/ __); + if (Result.first != HttpResponseCode::OK) + { + return {Result.first, + fmt::format("Chunk request for chunk {} in {}/{} failed. Reason: '{}'", ChunkId, ProjectId, OplogId, Result.second)}; + } + return Result; } std::pair<HttpResponseCode, std::string> @@ -4416,7 +4544,8 @@ ProjectStore::GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, ZenContentType AcceptType, - IoBuffer& OutChunk) + IoBuffer& OutChunk, + uint64_t* OptionalInOutModificationTag) { Ref<ProjectStore::Project> Project = OpenProject(ProjectId); if (!Project) @@ -4445,6 +4574,16 @@ ProjectStore::GetChunk(const std::string_view ProjectId, return {HttpResponseCode::NotFound, fmt::format("chunk - '{}' MISSING", Cid)}; } + if (OptionalInOutModificationTag != nullptr) + { + uint64_t OldTag = *OptionalInOutModificationTag; + *OptionalInOutModificationTag = GetModificationTagFromRawHash(Hash); + if (*OptionalInOutModificationTag == OldTag) + { + return {HttpResponseCode::NotModified, {}}; + } + } + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); @@ -4504,6 +4643,354 @@ ProjectStore::PutChunk(const std::string_view ProjectId, } std::pair<HttpResponseCode, std::string> +ProjectStore::GetChunks(const std::string_view ProjectId, + const std::string_view OplogId, + const CbObject& RequestObject, + CbPackage& OutResponsePackage) +{ + ZEN_TRACE_CPU("Store::GetChunks"); + + using namespace std::literals; + + Ref<ProjectStore::Project> Project = OpenProject(ProjectId); + if (!Project) + { + return {HttpResponseCode::NotFound, fmt::format("getchunks rpc request for unknown project '{}'", ProjectId)}; + } + Project->TouchProject(); + + ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return {HttpResponseCode::NotFound, fmt::format("getchunks rpc request for unknown oplog '{}/{}'", ProjectId, OplogId)}; + } + Project->TouchOplog(OplogId); + + if (RequestObject["chunks"sv].IsArray()) + { + // Legacy full chunks only by rawhash + + CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView(); + + CbObjectWriter ResponseWriter; + ResponseWriter.BeginArray("chunks"sv); + for (CbFieldView FieldView : ChunksArray) + { + IoHash RawHash = FieldView.AsHash(); + IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash); + if (ChunkBuffer) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer)); + if (Compressed) + { + ResponseWriter.AddHash(RawHash); + OutResponsePackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash)); + } + else + { + ZEN_WARN("oplog '{}/{}': invalid compressed binary in cas store for {}", ProjectId, OplogId, RawHash); + } + } + } + ResponseWriter.EndArray(); + OutResponsePackage.SetObject(ResponseWriter.Save()); + return {HttpResponseCode::OK, {}}; + } + else if (auto RequestFieldView = RequestObject["Request"sv]; RequestFieldView.IsObject()) + { + CbObjectView RequestView = RequestFieldView.AsObjectView(); + bool SkipData = RequestView["SkipData"].AsBool(false); + CbArrayView ChunksArray = RequestView["Chunks"sv].AsArrayView(); + + struct Request + { + struct InputData + { + uint64_t Offset = 0; + uint64_t Size = (uint64_t)-1; + std::variant<IoHash, Oid> Id; + std::optional<uint64_t> ModTag; + } Input; + struct OutputData + { + bool Exists = false; + IoBuffer ChunkBuffer; + uint64_t ModTag = 0; + } Output; + }; + + std::vector<Request> Requests; + size_t RequestCount = ChunksArray.Num(); + if (RequestCount > 0) + { + Requests.reserve(RequestCount); + std::vector<IoHash> ChunkRawHashes; + std::vector<size_t> ChunkRawHashesRequestIndex; + std::vector<Oid> ChunkIds; + std::vector<size_t> ChunkIdsRequestIndex; + bool DoBatch = RequestCount > 1; + if (DoBatch) + { + ChunkRawHashes.reserve(RequestCount); + ChunkRawHashesRequestIndex.reserve(RequestCount); + ChunkIds.reserve(RequestCount); + ChunkIdsRequestIndex.reserve(RequestCount); + } + for (CbFieldView FieldView : ChunksArray) + { + CbObjectView ChunkObject = FieldView.AsObjectView(); + Request ChunkRequest = { + .Input{.Offset = ChunkObject["Offset"sv].AsUInt64(0), .Size = ChunkObject["Size"sv].AsUInt64((uint64_t)-1)}}; + if (CbFieldView InputModificationTagView = ChunkObject.FindView("ModTag"); InputModificationTagView.IsInteger()) + { + ChunkRequest.Input.ModTag = InputModificationTagView.AsUInt64(); + } + if (CbFieldView RawHashView = ChunkObject.FindView("RawHash"sv); RawHashView.IsHash()) + { + const IoHash ChunkHash = RawHashView.AsHash(); + ChunkRequest.Input.Id = ChunkHash; + if (DoBatch) + { + ChunkRawHashes.push_back(ChunkHash); + ChunkRawHashesRequestIndex.push_back(Requests.size()); + } + } + else if (CbFieldView IdView = ChunkObject.FindView("Oid"sv); IdView.IsObjectId()) + { + const Oid ChunkId = IdView.AsObjectId(); + ChunkRequest.Input.Id = ChunkId; + if (DoBatch) + { + ChunkIds.push_back(ChunkId); + ChunkIdsRequestIndex.push_back(Requests.size()); + } + } + else + { + return {HttpResponseCode::BadRequest, + fmt::format("oplog '{}/{}': malformed getchunks rpc request object, chunk request has no identifier", + ProjectId, + OplogId)}; + } + Requests.emplace_back(std::move(ChunkRequest)); + } + + if (DoBatch) + { + WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); + if (!ChunkRawHashes.empty()) + { + FoundLog->IterateChunks( + ChunkRawHashes, + true, + [&](size_t Index, const IoBuffer& Payload, uint64_t ModTag) -> bool { + if (Payload) + { + size_t RequestIndex = ChunkRawHashesRequestIndex[Index]; + Requests[RequestIndex].Output.Exists = true; + if (!SkipData) + { + Requests[RequestIndex].Output.ChunkBuffer = Payload; + Requests[RequestIndex].Output.ChunkBuffer.MakeOwned(); + } + Requests[RequestIndex].Output.ModTag = ModTag; + } + return true; + }, + &WorkerPool, + 8u * 1024); + } + if (!ChunkIdsRequestIndex.empty()) + { + FoundLog->IterateChunks( + ChunkIds, + true, + [&](size_t Index, const IoBuffer& Payload, uint64_t ModTag) -> bool { + if (Payload) + { + size_t RequestIndex = ChunkIdsRequestIndex[Index]; + Requests[RequestIndex].Output.Exists = true; + if (!SkipData) + { + Requests[RequestIndex].Output.ChunkBuffer = Payload; + Requests[RequestIndex].Output.ChunkBuffer.MakeOwned(); + } + Requests[RequestIndex].Output.ModTag = ModTag; + } + return true; + }, + &WorkerPool, + 8u * 1024); + } + } + else + { + Request& ChunkRequest = Requests.front(); + if (ChunkRequest.Input.Id.index() == 0) + { + const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id); + IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash); + if (Payload) + { + ChunkRequest.Output.Exists = true; + ChunkRequest.Output.ModTag = GetModificationTagFromRawHash(ChunkHash); + if (!SkipData) + { + ChunkRequest.Output.ChunkBuffer = Payload; + } + } + } + else + { + const Oid& ChunkId = std::get<Oid>(ChunkRequest.Input.Id); + uint64_t ModTag = 0; + IoBuffer Payload = FoundLog->FindChunk(ChunkId, &ModTag); + if (Payload) + { + ChunkRequest.Output.Exists = true; + ChunkRequest.Output.ModTag = ModTag; + if (!SkipData) + { + ChunkRequest.Output.ChunkBuffer = Payload; + } + } + } + } + } + + CbObjectWriter ResponseWriter(32 + Requests.size() * 64u); + ResponseWriter.BeginArray("Chunks"sv); + { + for (Request& ChunkRequest : Requests) + { + if (ChunkRequest.Output.Exists) + { + ResponseWriter.BeginObject(); + { + if (ChunkRequest.Input.Id.index() == 0) + { + const IoHash& RawHash = std::get<IoHash>(ChunkRequest.Input.Id); + ResponseWriter.AddHash("Id", RawHash); + } + else + { + const Oid& Id = std::get<Oid>(ChunkRequest.Input.Id); + ResponseWriter.AddObjectId("Id", Id); + } + if (!ChunkRequest.Input.ModTag.has_value() || ChunkRequest.Input.ModTag.value() != ChunkRequest.Output.ModTag) + { + ResponseWriter.AddInteger("ModTag", ChunkRequest.Output.ModTag); + if (!SkipData) + { + CompositeBuffer ChunkRange; + ZenContentType ContentType; + IoHash FullChunkRawHash; + uint64_t FullChunkSize = 0; + auto ExtractRangeResult = ExtractRange(std::move(ChunkRequest.Output.ChunkBuffer), + ChunkRequest.Input.Offset, + ChunkRequest.Input.Size, + ZenContentType::kCompressedBinary, + ContentType, + ChunkRange, + FullChunkRawHash, + FullChunkSize); + if (ExtractRangeResult.first == HttpResponseCode::OK) + { + if (ContentType == ZenContentType::kCompressedBinary) + { + ZEN_ASSERT(FullChunkRawHash != IoHash::Zero); + CompressedBuffer CompressedValue = + CompressedBuffer::FromCompressedNoValidate(std::move(ChunkRange)); + ZEN_ASSERT(CompressedValue); + + if (FullChunkSize != 0) + { + // This really could use some thought so we don't send the same data if we get a request for + // multiple ranges from the same chunk block + + uint64_t FragmentRawOffset = 0; + OodleCompressor Compressor; + OodleCompressionLevel CompressionLevel; + uint64_t BlockSize = 0; + if (CompressedValue.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize)) + { + if (BlockSize > 0) + { + FragmentRawOffset = (ChunkRequest.Input.Offset / BlockSize) * BlockSize; + } + else + { + FragmentRawOffset = ChunkRequest.Input.Offset; + } + uint64_t FragmentRawLength = CompressedValue.DecodeRawSize(); + + IoHashStream FragmentHashStream; + FragmentHashStream.Append(FullChunkRawHash.Hash, sizeof(FullChunkRawHash.Hash)); + FragmentHashStream.Append(&FragmentRawOffset, sizeof(FragmentRawOffset)); + FragmentHashStream.Append(&FragmentRawLength, sizeof(FragmentRawLength)); + IoHash FragmentHash = FragmentHashStream.GetHash(); + + ResponseWriter.AddHash("FragmentHash", FragmentHash); + ResponseWriter.AddInteger("FragmentOffset", FragmentRawOffset); + ResponseWriter.AddInteger("RawSize", FullChunkSize); + OutResponsePackage.AddAttachment(CbAttachment(CompressedValue, FragmentHash)); + } + else + { + std::string ErrorString = + "Failed to get compression parameters from partial compressed buffer"; + ResponseWriter.AddString("Error", ErrorString); + ZEN_WARN("oplog '{}/{}': {}", ProjectId, OplogId, ErrorString); + } + } + else + { + ResponseWriter.AddHash("RawHash"sv, FullChunkRawHash); + OutResponsePackage.AddAttachment(CbAttachment(std::move(CompressedValue), FullChunkRawHash)); + } + } + else + { + IoHashStream HashStream; + ZEN_ASSERT(ChunkRequest.Input.Id.index() == 1); + const Oid& Id = std::get<Oid>(ChunkRequest.Input.Id); + HashStream.Append(Id.OidBits, sizeof(Id.OidBits)); + HashStream.Append(&ChunkRequest.Input.Offset, sizeof(ChunkRequest.Input.Offset)); + HashStream.Append(&ChunkRequest.Input.Size, sizeof(ChunkRequest.Input.Size)); + IoHash Hash = HashStream.GetHash(); + + ResponseWriter.AddHash("Hash"sv, Hash); + if (FullChunkSize != 0) + { + ResponseWriter.AddInteger("Size", FullChunkSize); + } + OutResponsePackage.AddAttachment(CbAttachment(std::move(ChunkRange), Hash)); + } + } + else + { + std::string ErrorString = fmt::format("Failed fetchiong chunk range ({})", ExtractRangeResult.second); + ResponseWriter.AddString("Error", ErrorString); + ZEN_WARN("oplog '{}/{}': {}", ProjectId, OplogId, ErrorString); + } + } + } + } + ResponseWriter.EndObject(); + } + } + } + ResponseWriter.EndArray(); + OutResponsePackage.SetObject(ResponseWriter.Save()); + return {HttpResponseCode::OK, {}}; + } + else + { + return {HttpResponseCode::BadRequest, fmt::format("oplog '{}/{}': malformed getchunks rpc request object", ProjectId, OplogId)}; + } +} + +std::pair<HttpResponseCode, std::string> ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_view OplogId, IoBuffer&& Payload, CbObject& OutResponse) { ZEN_TRACE_CPU("Store::WriteOplog"); @@ -4570,7 +5057,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie return ConvertResult(RemoteResult); } - CbObjectWriter Cbo; + CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1); Cbo.BeginArray("need"); { for (const IoHash& Hash : Attachments) @@ -4767,34 +5254,33 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, else if (Method == "getchunks"sv) { ZEN_TRACE_CPU("Store::Rpc::getchunks"); - CbPackage ResponsePackage; + + RpcAcceptOptions AcceptFlags = static_cast<RpcAcceptOptions>(Cb["AcceptFlags"sv].AsUInt16(0u)); + int32_t TargetProcessId = Cb["Pid"sv].AsInt32(0); + + CbPackage ResponsePackage; + std::pair<HttpResponseCode, std::string> Result = GetChunks(ProjectId, OplogId, Cb, ResponsePackage); + if (Result.first == HttpResponseCode::OK) { - CbArrayView ChunksArray = Cb["chunks"sv].AsArrayView(); - CbObjectWriter ResponseWriter; - ResponseWriter.BeginArray("chunks"sv); - for (CbFieldView FieldView : ChunksArray) + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) { - IoHash RawHash = FieldView.AsHash(); - IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash); - if (ChunkBuffer) + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer)); - if (Compressed) - { - ResponseWriter.AddHash(RawHash); - ResponsePackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash)); - } - else - { - ZEN_WARN("oplog '{}/{}': invalid compressed binary in cas store for {}", ProjectId, OplogId, RawHash); - } + Flags |= FormatFlags::kDenyPartialLocalReferences; } + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(HttpReq.SessionId(), TargetProcessId); } - ResponseWriter.EndArray(); - ResponsePackage.SetObject(ResponseWriter.Save()); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage, Flags, TargetProcessHandle); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + } + else + { + HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } - CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage, FormatFlags::kDefault); - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); return true; } else if (Method == "putchunks"sv) @@ -4927,7 +5413,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, ++TotalFiles; // Rewrite file array entry with new data reference - CbObjectWriter Writer; + CbObjectWriter Writer(View.GetSize()); RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { if (Field.GetName() == "data"sv) { @@ -6012,7 +6498,7 @@ namespace testutils { return OidStringBuilder.ToString(); } - CbPackage CreateOplogPackage(const Oid& Id, const std::span<const std::pair<Oid, CompressedBuffer>>& Attachments) + CbPackage CreateBulkDataOplogPackage(const Oid& Id, const std::span<const std::pair<Oid, CompressedBuffer>>& Attachments) { CbPackage Package; CbObjectWriter Object; @@ -6038,16 +6524,43 @@ namespace testutils { return Package; }; + CbPackage CreateFilesOplogPackage(const Oid& Id, + const std::filesystem::path ProjectRootDir, + const std::span<const std::pair<Oid, std::filesystem::path>>& Attachments) + { + CbPackage Package; + CbObjectWriter Object; + Object << "key"sv << OidAsString(Id); + if (!Attachments.empty()) + { + Object.BeginArray("files"); + for (const auto& Attachment : Attachments) + { + std::filesystem::path ServerPath = std::filesystem::relative(Attachment.second, ProjectRootDir); + std::filesystem::path ClientPath = ServerPath; // dummy + Object.BeginObject(); + Object << "id"sv << Attachment.first; + Object << "serverpath"sv << ServerPath.string(); + Object << "clientpath"sv << ClientPath.string(); + Object.EndObject(); + } + Object.EndArray(); + } + Package.SetObject(Object.Save()); + return Package; + }; + std::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments( const std::span<const size_t>& Sizes, - OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast) + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0) { std::vector<std::pair<Oid, CompressedBuffer>> Result; Result.reserve(Sizes.size()); for (size_t Size : Sizes) { CompressedBuffer Compressed = - CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(Size)), OodleCompressor::Mermaid, CompressionLevel); + CompressedBuffer::Compress(SharedBuffer(CreateSemiRandomBlob(Size)), OodleCompressor::Mermaid, CompressionLevel, BlockSize); Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed)); } return Result; @@ -6069,6 +6582,63 @@ namespace testutils { return 0; } + template<typename ChunkType> + CbObject BuildChunksRequest(bool SkipData, + std::string_view IdName, + const std::vector<ChunkType>& Chunks, + const std::vector<std::pair<size_t, size_t>>& Ranges, + const std::vector<uint64_t>& ModTags) + { + CbObjectWriter Request; + Request.BeginObject("Request"sv); + { + if (SkipData) + { + Request.AddBool("SkipData"sv, true); + } + if (!Chunks.empty()) + { + Request.BeginArray("Chunks"); + for (size_t Index = 0; Index < Chunks.size(); Index++) + { + Request.BeginObject(); + { + Request << IdName << Chunks[Index]; + if (!ModTags.empty()) + { + Request << "ModTag" << ModTags[Index]; + } + if (!Ranges.empty()) + { + Request << "Offset" << Ranges[Index].first; + Request << "Size" << Ranges[Index].second; + } + } + Request.EndObject(); + } + Request.EndArray(); + } + } + Request.EndObject(); + return Request.Save(); + }; + + CbObject BuildChunksRequest(bool SkipData, + const std::vector<Oid>& Chunks, + const std::vector<std::pair<size_t, size_t>>& Ranges, + const std::vector<uint64_t>& ModTags) + { + return BuildChunksRequest<Oid>(SkipData, "Oid", Chunks, Ranges, ModTags); + } + + CbObject BuildChunksRequest(bool SkipData, + const std::vector<IoHash>& Chunks, + const std::vector<std::pair<size_t, size_t>>& Ranges, + const std::vector<uint64_t>& ModTags) + { + return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags); + } + } // namespace testutils TEST_CASE("project.store.create") @@ -6078,6 +6648,7 @@ TEST_CASE("project.store.create") ScopedTemporaryDirectory TempDir; auto JobQueue = MakeJobQueue(1, ""sv); + OpenProcessCache ProcessCache; GcManager Gc; CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; @@ -6085,7 +6656,7 @@ TEST_CASE("project.store.create") std::string_view ProjectName("proj1"sv); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -6108,13 +6679,14 @@ TEST_CASE("project.store.lifetimes") ScopedTemporaryDirectory TempDir; auto JobQueue = MakeJobQueue(1, ""sv); + OpenProcessCache ProcessCache; GcManager Gc; CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -6170,13 +6742,14 @@ TEST_CASE_TEMPLATE("project.store.export", ScopedTemporaryDirectory ExportDir; auto JobQueue = MakeJobQueue(1, ""sv); + OpenProcessCache ProcessCache; GcManager Gc; CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -6191,13 +6764,14 @@ TEST_CASE_TEMPLATE("project.store.export", ProjectStore::Oplog* Oplog = Project->NewOplog("oplog1", {}); CHECK(Oplog != nullptr); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); Oplog->AppendNewOplogEntry( - CreateOplogPackage(Oid::NewOid(), - CreateAttachments(std::initializer_list<size_t>{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None))); + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage( + Oid::NewOid(), + CreateAttachments(std::initializer_list<size_t>{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None))); FileRemoteStoreOptions Options = { RemoteStoreOptions{.MaxBlockSize = 64u * 1024, .MaxChunkEmbedSize = 32 * 1024u, .ChunkFileSizeLimit = 64u * 1024u}, @@ -6271,13 +6845,14 @@ TEST_CASE("project.store.gc") ScopedTemporaryDirectory TempDir; auto JobQueue = MakeJobQueue(1, ""sv); + OpenProcessCache ProcessCache; GcManager Gc; CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -6325,10 +6900,11 @@ TEST_CASE("project.store.gc") ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1", Project1OplogPath); CHECK(Oplog != nullptr); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); + Oplog->AppendNewOplogEntry( + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); } { @@ -6342,21 +6918,23 @@ TEST_CASE("project.store.gc") ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog2", Project2Oplog1Path); CHECK(Oplog != nullptr); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177}))); Oplog->AppendNewOplogEntry( - CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221}))); + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96}))); + Oplog->AppendNewOplogEntry( + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221}))); } { ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog3", Project2Oplog2Path); CHECK(Oplog != nullptr); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{137}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{137}))); + Oplog->AppendNewOplogEntry( + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9723, 683, 594, 98}))); Oplog->AppendNewOplogEntry( - CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9723, 683, 594, 98}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{531, 271}))); + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{531, 271}))); } } @@ -6468,13 +7046,14 @@ TEST_CASE("project.store.gc.prep") ScopedTemporaryDirectory TempDir; auto JobQueue = MakeJobQueue(1, ""sv); + OpenProcessCache ProcessCache; GcManager Gc; CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -6507,7 +7086,7 @@ TEST_CASE("project.store.gc.prep") Project1RootDir.string(), Project1FilePath.string())); ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments)); } { Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); @@ -6538,7 +7117,7 @@ TEST_CASE("project.store.gc.prep") // Make sure the chunks are stored but not the referencing op Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments)); Project1->DeleteOplog("oplog1"sv); } { @@ -6585,7 +7164,7 @@ TEST_CASE("project.store.gc.prep") { Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); ProjectStore::Oplog* Oplog = Project1->OpenOplog("oplog1"sv, true, true); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments)); Oplog->RemovePendingChunkReferences(OpChunkHashes); CHECK(Oplog->GetPendingChunkReferencesLocked().size() == 0); } @@ -6619,7 +7198,7 @@ TEST_CASE("project.store.gc.prep") // Make sure the chunks are stored but not the referencing op Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments)); Project1->DeleteOplog("oplog1"sv); } @@ -6674,6 +7253,928 @@ TEST_CASE("project.store.gc.prep") } } +TEST_CASE("project.store.rpc.getchunks") +{ + using namespace std::literals; + using namespace testutils; + + ScopedTemporaryDirectory TempDir; + + auto JobQueue = MakeJobQueue(1, ""sv); + OpenProcessCache ProcessCache; + GcManager Gc; + CidStore CidStore(Gc); + CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; + CidStore.Initialize(CidConfig); + + std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); + std::filesystem::path RootDir = TempDir.Path() / "root"sv; + std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; + + std::filesystem::path Project1RootDir = TempDir.Path() / "game1"sv; + std::filesystem::path Project1FilePath = TempDir.Path() / "game1"sv / "game.uproject"sv; + { + CreateDirectories(Project1FilePath.parent_path()); + BasicFile ProjectFile; + ProjectFile.Open(Project1FilePath, BasicFile::Mode::kTruncate); + } + + std::vector<Oid> OpIds; + OpIds.insert(OpIds.end(), {Oid::NewOid(), Oid::NewOid(), Oid::NewOid(), Oid::NewOid()}); + std::unordered_map<Oid, std::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; + Oid FilesOpId = Oid::NewOid(); + std::vector<std::pair<Oid, std::filesystem::path>> FilesOpIdAttachments; + { + Ref<ProjectStore::Project> Project1(ProjectStore.NewProject(BasePath / "proj1"sv, + "proj1"sv, + RootDir.string(), + EngineRootDir.string(), + Project1RootDir.string(), + Project1FilePath.string())); + ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, {}); + CHECK(Oplog != nullptr); + Attachments[OpIds[0]] = {}; + Attachments[OpIds[1]] = CreateAttachments(std::initializer_list<size_t>{77}); + Attachments[OpIds[2]] = + CreateAttachments(std::initializer_list<size_t>{200 * 1024, 314 * 1024, 690, 99}, OodleCompressionLevel::VeryFast, 128 * 1024); + Attachments[OpIds[3]] = CreateAttachments(std::initializer_list<size_t>{55, 122}); + for (auto It : Attachments) + { + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(It.first, It.second)); + } + + std::filesystem::path UncompressedFilePath = RootDir / "content" / "uncompressed_file"; + CreateDirectories(UncompressedFilePath.parent_path()); + IoBuffer FileBlob = CreateRandomBlob(81823 * 2); + WriteFile(UncompressedFilePath, FileBlob); + FilesOpIdAttachments.push_back({Oid::NewOid(), UncompressedFilePath}); + Oplog->AppendNewOplogEntry(CreateFilesOplogPackage(FilesOpId, RootDir, FilesOpIdAttachments)); + } + + // Invalid request + { + CbObjectWriter Request; + Request.BeginObject("WrongName"sv); + Request.EndObject(); + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, "oplog1"sv, Request.Save(), Response); + CHECK_EQ(HttpResponseCode::BadRequest, Result.first); + } + + // Empty request + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, std::vector<IoHash>{}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + // Single non-existing chunk by RawHash + IoHash NotFoundIoHash = IoHash::Max; + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {NotFoundIoHash}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {NotFoundIoHash}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + // Single non-existing chunk by Id + Oid NotFoundId = Oid::NewOid(); + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {NotFoundId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, "oplog1"sv, testutils::BuildChunksRequest(/*SkipData*/ true, {NotFoundId}, {}, {}), Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + // Single existing chunk by RawHash + { + // Fresh fetch + IoHash FirstAttachmentHash = Attachments[OpIds[2]][1].second.DecodeRawHash(); + uint64_t ResponseModTag = 0; + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fetch with matching ModTag + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fetch with mismatching ModTag + { + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fresh modtime query + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with matching ModTag + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with mismatching ModTag + { + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + // Single existing CID chunk by Id + { + Oid FirstAttachmentId = Attachments[OpIds[2]][1].first; + uint64_t ResponseModTag = 0; + { + // Full chunk request + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Partial chunk request + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {{130 * 1024, 8100}}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["FragmentHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + std::uint64_t FragmentStart = Chunk["FragmentOffset"].AsUInt64(); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK(FragmentStart <= 130 * 1024); + CHECK(FragmentStart + Buffer.DecodeRawSize() >= 130 * 1024 + 8100); + auto ResponseDecompressedBuffer = Buffer.Decompress(130 * 1024 - FragmentStart, 8100); + auto ExpectedDecompressedBuffer = Attachments[OpIds[2]][1].second.Decompress(130 * 1024, 8100); + CHECK(ResponseDecompressedBuffer.AsIoBuffer().GetView().EqualBytes(ExpectedDecompressedBuffer.AsIoBuffer().GetView())); + CHECK_EQ(Chunk["RawSize"sv].AsUInt64(), Attachments[OpIds[2]][1].second.DecodeRawSize()); + CHECK(!Chunk.FindView("Size")); + } + { + // Fetch with matching ModTag + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Fetch with mismatching ModTag + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag3 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag3); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fresh modtime query + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with matching ModTag + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with mismatching ModTag + { + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + + // Single existing file chunk by Id + { + Oid FirstAttachmentId = FilesOpIdAttachments[0].first; + uint64_t ResponseModTag = 0; + { + // Full chunk request + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["Hash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompositeBuffer Buffer = Attachment->AsCompositeBinary(); + CHECK_EQ(IoHash::HashBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten()), IoHash::HashBuffer(Buffer)); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Partial chunk request + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {{81823, 5434}}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + + IoHash AttachmentHash = Chunk["Hash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompositeBuffer Buffer = Attachment->AsCompositeBinary(); + CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)), + IoHash::HashBuffer(Buffer)); + CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second)); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Fetch with matching ModTag + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("Hash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Fetch with mismatching ModTag + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag3 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag3); + IoHash AttachmentHash = Chunk["Hash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompositeBuffer Buffer = Attachment->AsCompositeBinary(); + CHECK_EQ(IoHash::HashBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten()), IoHash::HashBuffer(Buffer)); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fresh modtime query + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with matching ModTag + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("Hash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with mismatching ModTag + { + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + + // Multi RawHash Request + { + std::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second, + Attachments[OpIds[2]][0].second, + Attachments[OpIds[2]][1].second}; + std::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(), + AttachmentBuffers[1].DecodeRawHash(), + AttachmentBuffers[2].DecodeRawHash()}; + std::vector<uint64_t> ResponseModTags(3, 0); + { + // Fresh fetch + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, AttachmentHashes, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(3, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + ResponseModTags[Index] = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTags[Index]); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(AttachmentBuffers[Index].GetCompressed().Flatten().GetView().EqualBytes(Buffer.GetCompressed().Flatten().GetView())); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Fetch with matching ModTag + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, AttachmentHashes, {}, ResponseModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Fresh modtime query + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + CHECK_EQ(ResponseModTags[Index], Chunk["ModTag"].AsUInt64()); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Modtime query with matching ModTags + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, ResponseModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Modtime query with mismatching ModTags + std::vector<uint64_t> MismatchingModTags(ResponseModTags); + for (uint64_t& Tag : MismatchingModTags) + { + Tag++; + } + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, MismatchingModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + CHECK(Chunk["ModTag"].AsUInt64() == ResponseModTags[Index]); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + } + // Multi Id Request + { + std::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second, + Attachments[OpIds[2]][0].second, + Attachments[OpIds[2]][1].second}; + std::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(), + AttachmentBuffers[1].DecodeRawHash(), + AttachmentBuffers[2].DecodeRawHash()}; + std::vector<Oid> AttachedIds{Attachments[OpIds[1]][0].first, Attachments[OpIds[2]][0].first, Attachments[OpIds[2]][1].first}; + std::vector<uint64_t> ResponseModTags(3, 0); + { + // Fresh fetch + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, AttachedIds, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(3, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + ResponseModTags[Index] = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTags[Index]); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(AttachmentBuffers[Index].GetCompressed().Flatten().GetView().EqualBytes(Buffer.GetCompressed().Flatten().GetView())); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Fetch with matching ModTag + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, AttachedIds, {}, ResponseModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Fresh modtime query + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + CHECK_EQ(ResponseModTags[Index], Chunk["ModTag"].AsUInt64()); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Modtime query with matching ModTags + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, ResponseModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Modtime query with mismatching ModTags + std::vector<uint64_t> MismatchingModTags(ResponseModTags); + for (uint64_t& Tag : MismatchingModTags) + { + Tag++; + } + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, MismatchingModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + CHECK(Chunk["ModTag"].AsUInt64() == ResponseModTags[Index]); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + } +} + TEST_CASE("project.store.partial.read") { using namespace std::literals; @@ -6682,13 +8183,14 @@ TEST_CASE("project.store.partial.read") ScopedTemporaryDirectory TempDir; auto JobQueue = MakeJobQueue(1, ""sv); + OpenProcessCache ProcessCache; GcManager Gc; CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; @@ -6718,63 +8220,119 @@ TEST_CASE("project.store.partial.read") Attachments[OpIds[3]] = CreateAttachments(std::initializer_list<size_t>{55, 122}); for (auto It : Attachments) { - Oplog->AppendNewOplogEntry(CreateOplogPackage(It.first, It.second)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(It.first, It.second)); } } { + uint64_t ModificationTag = 0; IoBuffer Chunk; CHECK(ProjectStore .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), HttpContentType::kCompressedBinary, - Chunk) + Chunk, + &ModificationTag) .first == HttpResponseCode::OK); IoHash RawHash; uint64_t RawSize; CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize); CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize()); - } - CompositeBuffer ChunkResult; - HttpContentType ContentType; - CHECK(ProjectStore - .GetChunkRange("proj1"sv, - "oplog1"sv, - OidAsString(Attachments[OpIds[2]][1].first), - 0, - ~0ull, - HttpContentType::kCompressedBinary, - ChunkResult, - ContentType) - .first == HttpResponseCode::OK); - CHECK(ChunkResult); - CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() == - Attachments[OpIds[2]][1].second.DecodeRawSize()); - - CompositeBuffer PartialChunkResult; - CHECK(ProjectStore - .GetChunkRange("proj1"sv, - "oplog1"sv, - OidAsString(Attachments[OpIds[2]][1].first), - 5, - 1773, - HttpContentType::kCompressedBinary, - PartialChunkResult, - ContentType) - .first == HttpResponseCode::OK); - CHECK(PartialChunkResult); - IoHash PartialRawHash; - uint64_t PartialRawSize; - CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize); - CHECK(PartialRawSize >= 1773); - - uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5); - SharedBuffer PartialDecompressed = PartialCompressedResult.Decompress(RawOffsetInPartialCompressed); - SharedBuffer FullDecompressed = Attachments[OpIds[2]][1].second.Decompress(); - const uint8_t* FullDataPtr = &(reinterpret_cast<const uint8_t*>(FullDecompressed.GetView().GetData())[5]); - const uint8_t* PartialDataPtr = reinterpret_cast<const uint8_t*>(PartialDecompressed.GetView().GetData()); - CHECK(FullDataPtr[0] == PartialDataPtr[0]); + CHECK(ModificationTag != 0); + CHECK(ProjectStore + .GetChunk("proj1"sv, + "oplog1"sv, + Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), + HttpContentType::kCompressedBinary, + Chunk, + &ModificationTag) + .first == HttpResponseCode::NotModified); + } + + { + uint64_t FullChunkModificationTag = 0; + { + CompositeBuffer ChunkResult; + HttpContentType ContentType; + CHECK(ProjectStore + .GetChunkRange("proj1"sv, + "oplog1"sv, + OidAsString(Attachments[OpIds[2]][1].first), + 0, + ~0ull, + HttpContentType::kCompressedBinary, + ChunkResult, + ContentType, + &FullChunkModificationTag) + .first == HttpResponseCode::OK); + CHECK(ChunkResult); + CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() == + Attachments[OpIds[2]][1].second.DecodeRawSize()); + } + { + CompositeBuffer ChunkResult; + HttpContentType ContentType; + CHECK(ProjectStore + .GetChunkRange("proj1"sv, + "oplog1"sv, + OidAsString(Attachments[OpIds[2]][1].first), + 0, + ~0ull, + HttpContentType::kCompressedBinary, + ChunkResult, + ContentType, + &FullChunkModificationTag) + .first == HttpResponseCode::NotModified); + } + } + { + CompositeBuffer PartialChunkResult; + uint64_t PartialChunkModificationTag = 0; + { + CompositeBuffer ChunkResult; + HttpContentType ContentType; + CHECK(ProjectStore + .GetChunkRange("proj1"sv, + "oplog1"sv, + OidAsString(Attachments[OpIds[2]][1].first), + 5, + 1773, + HttpContentType::kCompressedBinary, + PartialChunkResult, + ContentType, + &PartialChunkModificationTag) + .first == HttpResponseCode::OK); + CHECK(PartialChunkResult); + IoHash PartialRawHash; + uint64_t PartialRawSize; + CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize); + CHECK(PartialRawSize >= 1773); + + uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5); + SharedBuffer PartialDecompressed = PartialCompressedResult.Decompress(RawOffsetInPartialCompressed); + SharedBuffer FullDecompressed = Attachments[OpIds[2]][1].second.Decompress(); + const uint8_t* FullDataPtr = &(reinterpret_cast<const uint8_t*>(FullDecompressed.GetView().GetData())[5]); + const uint8_t* PartialDataPtr = reinterpret_cast<const uint8_t*>(PartialDecompressed.GetView().GetData()); + CHECK(FullDataPtr[0] == PartialDataPtr[0]); + } + + { + CompositeBuffer ChunkResult; + HttpContentType ContentType; + CHECK(ProjectStore + .GetChunkRange("proj1"sv, + "oplog1"sv, + OidAsString(Attachments[OpIds[2]][1].first), + 5, + 1773, + HttpContentType::kCompressedBinary, + PartialChunkResult, + ContentType, + &PartialChunkModificationTag) + .first == HttpResponseCode::NotModified); + } + } } TEST_CASE("project.store.block") @@ -6808,13 +8366,14 @@ TEST_CASE("project.store.iterateoplog") ScopedTemporaryDirectory TempDir; auto JobQueue = MakeJobQueue(1, ""sv); + OpenProcessCache ProcessCache; GcManager Gc; CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv"; @@ -6851,7 +8410,7 @@ TEST_CASE("project.store.iterateoplog") TestOidData TestOids[NumTestOids]; for (const TestOidData& TestOid : TestOids) { - Oplog->AppendNewOplogEntry(CreateOplogPackage(TestOid.KeyAsOidNotOplogId, {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(TestOid.KeyAsOidNotOplogId, {})); } int Count = 0; |