diff options
| author | Dan Engelbrecht <[email protected]> | 2024-12-05 14:21:12 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2024-12-05 14:21:12 +0100 |
| commit | d1c9573ef4b54e03b66a9507a57104591078d35b (patch) | |
| tree | 38dbcbe99a5b76d322d1634014bc390e75b01c20 /src | |
| parent | Unity build fixes (#253) (diff) | |
| download | zen-d1c9573ef4b54e03b66a9507a57104591078d35b.tar.xz zen-d1c9573ef4b54e03b66a9507a57104591078d35b.zip | |
projectstore getchunks rpc with modtag (#244)
Feature: Project store "getchunks" rpc call /prj/{project}/oplog/{log}/rpc extended to accept both CAS (RawHash) and Id (Oid) identifiers as well as partial ranges
Diffstat (limited to 'src')
| -rw-r--r-- | src/zencore/filesystem.cpp | 24 | ||||
| -rw-r--r-- | src/zencore/include/zencore/compactbinarypackage.h | 4 | ||||
| -rw-r--r-- | src/zencore/include/zencore/filesystem.h | 4 | ||||
| -rw-r--r-- | src/zencore/include/zencore/testutils.h | 1 | ||||
| -rw-r--r-- | src/zencore/testutils.cpp | 20 | ||||
| -rw-r--r-- | src/zenserver/projectstore/httpprojectstore.cpp | 12 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 1931 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.h | 34 | ||||
| -rw-r--r-- | src/zenserver/projectstore/zenremoteprojectstore.cpp | 16 | ||||
| -rw-r--r-- | src/zenserver/vfs/vfsimpl.cpp | 3 | ||||
| -rw-r--r-- | src/zenstore/compactcas.cpp | 32 |
11 files changed, 1848 insertions, 233 deletions
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 36147c5a9..52f2c4adc 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -1435,12 +1435,34 @@ FileSizeFromHandle(void* NativeHandle) int Fd = int(intptr_t(NativeHandle)); struct stat Stat; fstat(Fd, &Stat); - FileSize = size_t(Stat.st_size); + FileSize = size_t(Stat.st_size); #endif return FileSize; } +uint64_t +GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + FILETIME LastWriteTime; + BOOL OK = GetFileTime((HANDLE)NativeHandle, NULL, NULL, &LastWriteTime); + if (OK) + { + return ((uint64_t(LastWriteTime.dwHighDateTime) << 32) | LastWriteTime.dwLowDateTime); + } +#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + int Fd = int(uintptr_t(NativeHandle)); + struct stat Stat; + if (0 == fstat(Fd, &Stat)) + { + return gsl::narrow<uint64_t>(Stat.st_mtime); + } +#endif + Ec = MakeErrorCodeFromLastError(); + return 0; +} + std::filesystem::path GetRunningExecutablePath() { diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h index 064481f83..12fcc41b7 100644 --- a/src/zencore/include/zencore/compactbinarypackage.h +++ b/src/zencore/include/zencore/compactbinarypackage.h @@ -64,6 +64,9 @@ public: ZENCORE_API CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash); ZENCORE_API CbAttachment(CompressedBuffer&& InValue, const IoHash& Hash); + /** Construct a binary attachment. Value is cloned if not owned. */ + ZENCORE_API CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash); + /** Reset this to a null attachment. */ inline void Reset() { *this = CbAttachment(); } @@ -130,7 +133,6 @@ public: private: ZENCORE_API CbAttachment(const CbObject& Value, const IoHash* Hash); ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue); - ZENCORE_API CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash); IoHash Hash; std::variant<std::nullptr_t, CbObject, CompositeBuffer, CompressedBuffer> Value; diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 2cd663afb..dba4981f0 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -45,6 +45,10 @@ ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, st */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); +/** Get a native time tick of last modification time + */ +ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); + ZENCORE_API std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h index 6a1c0184b..45fde4eda 100644 --- a/src/zencore/include/zencore/testutils.h +++ b/src/zencore/include/zencore/testutils.h @@ -33,6 +33,7 @@ struct ScopedCurrentDirectoryChange }; IoBuffer CreateRandomBlob(uint64_t Size); +IoBuffer CreateSemiRandomBlob(uint64_t Size); struct FalseType { diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp index d4c8aeaef..641d5508a 100644 --- a/src/zencore/testutils.cpp +++ b/src/zencore/testutils.cpp @@ -71,6 +71,26 @@ CreateRandomBlob(uint64_t Size) return Data; }; +IoBuffer +CreateSemiRandomBlob(uint64_t Size) +{ + IoBuffer Result(Size); + const size_t PartCount = (Size / (1u * 1024u * 64)) + 1; + const size_t PartSize = Size / PartCount; + auto Part = CreateRandomBlob(PartSize); + auto Remain = Result.GetMutableView().CopyFrom(Part.GetView()); + while (Remain.GetSize() >= PartSize) + { + Remain = Remain.CopyFrom(Part.GetView()); + } + if (Remain.GetSize() > 0) + { + auto RemainBuffer = CreateRandomBlob(Remain.GetSize()); + Remain.CopyFrom(RemainBuffer.GetView()); + } + return Result; +}; + } // namespace zen #endif // ZEN_WITH_TESTS diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 22216e88d..09f4c7ee2 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -561,7 +561,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) { const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; - IoBuffer FoundChunk = FoundLog->FindChunk(RequestedChunk.ChunkId); + IoBuffer FoundChunk = FoundLog->FindChunk(RequestedChunk.ChunkId, nullptr); if (FoundChunk) { if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1)) @@ -834,7 +834,7 @@ HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req) CompositeBuffer Chunk; HttpContentType ContentType; std::pair<HttpResponseCode, std::string> Result = - m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk, ContentType); + m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk, ContentType, nullptr); if (Result.first == HttpResponseCode::OK) { m_ProjectStats.ChunkHitCount++; @@ -883,8 +883,8 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) case HttpVerb::kGet: { IoBuffer Value; - std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value); - + std::pair<HttpResponseCode, std::string> Result = + m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr); if (Result.first == HttpResponseCode::OK) { m_ProjectStats.ChunkHitCount++; @@ -991,7 +991,7 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) std::vector<IoHash> NeedList = FoundLog->CheckPendingChunkReferences(ChunkList, std::chrono::minutes(2)); - CbObjectWriter Cbo; + CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1); Cbo.BeginArray("need"); { for (const IoHash& Hash : NeedList) @@ -1544,7 +1544,7 @@ LoadReferencedSet(ProjectStore::Oplog& Log) return std::optional<OplogReferencedSet>(); } - return OplogReferencedSet::LoadFromChunk(Log.FindChunk(ChunkId)); + return OplogReferencedSet::LoadFromChunk(Log.FindChunk(ChunkId, nullptr)); } void diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 415c80078..cd0b53394 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -314,6 +314,27 @@ namespace { static_assert(sizeof(OplogIndexHeader) == 64); + static std::uint64_t GetModificationTagFromRawHash(const IoHash& Hash) + { + IoHash::Hasher H; + return H(Hash); + } + + static std::uint64_t GetModificationTagFromModificationTime(IoBuffer FileBuffer) + { + IoBufferFileReference FileRef; + if (FileBuffer.GetFileReference(FileRef)) + { + std::error_code Ec; + uint64_t ModificationTick = GetModificationTickFromHandle(FileRef.FileHandle, Ec); + if (!Ec) + { + return ModificationTick; + } + } + return {}; + } + } // namespace ////////////////////////////////////////////////////////////////////////// @@ -1920,19 +1941,27 @@ ProjectStore::Oplog::GetChunkByRawHash(const IoHash& RawHash) } bool -ProjectStore::Oplog::IterateChunks(std::span<IoHash> RawHashes, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit) -{ - return m_CidStore.IterateChunks(RawHashes, AsyncCallback, OptionalWorkerPool, LargeSizeLimit); +ProjectStore::Oplog::IterateChunks(std::span<IoHash> RawHashes, + bool IncludeModTag, + const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) +{ + return m_CidStore.IterateChunks( + RawHashes, + [&](size_t Index, const IoBuffer& Payload) { + return AsyncCallback(Index, Payload, IncludeModTag ? GetModificationTagFromRawHash(RawHashes[Index]) : 0); + }, + OptionalWorkerPool, + LargeSizeLimit); } bool -ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit) +ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, + bool IncludeModTag, + const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) { std::vector<size_t> CidChunkIndexes; std::vector<IoHash> CidChunkHashes; @@ -1960,12 +1989,6 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } } } - m_CidStore.IterateChunks( - CidChunkHashes, - [&](size_t Index, const IoBuffer& Payload) { return AsyncCallback(CidChunkIndexes[Index], Payload); }, - OptionalWorkerPool, - LargeSizeLimit); - if (OptionalWorkerPool) { std::atomic_bool Result = true; @@ -1979,7 +2002,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } WorkLatch.AddCount(1); OptionalWorkerPool->ScheduleWork( - [this, &WorkLatch, &ChunkIds, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() { + [this, &WorkLatch, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() { auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); if (Result.load() == false) { @@ -1995,7 +2018,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[ChunkIndex], FilePath); } - if (!AsyncCallback(FileChunkIndex, Payload)) + if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0)) { Result.store(false); } @@ -2012,6 +2035,18 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, }); } + if (!CidChunkHashes.empty()) + { + m_CidStore.IterateChunks( + CidChunkHashes, + [&](size_t Index, const IoBuffer& Payload) { + size_t CidChunkIndex = CidChunkIndexes[Index]; + return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0); + }, + OptionalWorkerPool, + LargeSizeLimit); + } + WorkLatch.CountDown(); WorkLatch.Wait(); @@ -2019,6 +2054,18 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } else { + if (!CidChunkHashes.empty()) + { + m_CidStore.IterateChunks( + CidChunkHashes, + [&](size_t Index, const IoBuffer& Payload) { + size_t CidChunkIndex = CidChunkIndexes[Index]; + return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0); + }, + OptionalWorkerPool, + LargeSizeLimit); + } + for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++) { size_t FileChunkIndex = FileChunkIndexes[ChunkIndex]; @@ -2026,7 +2073,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath); if (Payload) { - bool Result = AsyncCallback(FileChunkIndex, Payload); + bool Result = AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0); if (!Result) { return false; @@ -2038,7 +2085,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } IoBuffer -ProjectStore::Oplog::FindChunk(const Oid& ChunkId) +ProjectStore::Oplog::FindChunk(const Oid& ChunkId, uint64_t* OptOutModificationTag) { RwLock::SharedLockScope OplogLock(m_OplogLock); if (!m_Storage) @@ -2051,7 +2098,12 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId) IoHash ChunkHash = ChunkIt->second; OplogLock.ReleaseNow(); - return m_CidStore.FindChunkByCid(ChunkHash); + IoBuffer Result = m_CidStore.FindChunkByCid(ChunkHash); + if (Result && OptOutModificationTag != nullptr) + { + *OptOutModificationTag = GetModificationTagFromRawHash(ChunkHash); + } + return Result; } if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end()) @@ -2065,6 +2117,10 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId) { ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkId, FilePath); } + else if (OptOutModificationTag != nullptr) + { + *OptOutModificationTag = GetModificationTagFromModificationTime(Result); + } return Result; } @@ -2073,7 +2129,12 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId) IoHash ChunkHash = MetaIt->second; OplogLock.ReleaseNow(); - return m_CidStore.FindChunkByCid(ChunkHash); + IoBuffer Result = m_CidStore.FindChunkByCid(ChunkHash); + if (Result && OptOutModificationTag != nullptr) + { + *OptOutModificationTag = GetModificationTagFromRawHash(ChunkHash); + } + return Result; } return {}; @@ -2109,7 +2170,7 @@ ProjectStore::Oplog::GetAllChunksInfo() for (ChunkInfo& Info : InfoArray) { - if (IoBuffer Chunk = FindChunk(Info.ChunkId)) + if (IoBuffer Chunk = FindChunk(Info.ChunkId, nullptr)) { Info.ChunkSize = Chunk.GetSize(); } @@ -3032,7 +3093,7 @@ ProjectStore::Project::WriteAccessTimes() { using namespace std::literals; - CbObjectWriter Writer; + CbObjectWriter Writer(32 + (m_LastAccessTimes.size() * 16)); { RwLock::SharedLockScope _(m_LastAccessTimesLock); @@ -3972,7 +4033,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, FoundLog->IterateChunks( Ids, - [&](size_t Index, const IoBuffer& Payload) { + false, + [&](size_t Index, const IoBuffer& Payload, uint64_t /*ModTag*/) { try { if (Payload) @@ -4033,7 +4095,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, 256u * 1024u); } - CbObjectWriter Response; + CbObjectWriter Response(64u + Count * ((WantsIdField ? (5 + sizeof(Oid::OidBits)) : 0) + (WantsServerPathField ? 64 : 0) + + (WantsClientPathField ? 64 : 0) + (WantsSizeField ? 16 : 0) + (WantsRawSizeField ? 16 : 0))); Response.BeginArray("files"sv); for (size_t Index = 0; Index < Count; Index++) { @@ -4102,7 +4165,16 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, std::vector<uint64_t> RawSizes; std::vector<uint64_t> Sizes; - size_t Count = 0; + size_t Count = 0; + size_t EstimatedCount = FoundLog->OplogCount(); + if (WantsIdField) + { + Ids.reserve(EstimatedCount); + } + if (WantsRawHashField || WantsRawSizeField || WantsSizeField) + { + Hashes.reserve(EstimatedCount); + } FoundLog->IterateChunkMap([&](const Oid& Id, const IoHash& Hash) { if (WantsIdField) { @@ -4129,7 +4201,8 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); (void)FoundLog->IterateChunks( Hashes, - [&](size_t Index, const IoBuffer& Payload) -> bool { + false, + [&](size_t Index, const IoBuffer& Payload, uint64_t /*ModTag*/) -> bool { try { if (Payload) @@ -4190,7 +4263,9 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, 256u * 1024u); } - CbObjectWriter Response; + CbObjectWriter Response(64u + Count * ((WantsIdField ? (5 + sizeof(Oid::OidBits)) : 0) + + (WantsRawHashField ? (10 + sizeof(IoHash::Hash)) : 0) + (WantsSizeField ? 16 : 0) + + (WantsRawSizeField ? 16 : 0))); Response.BeginArray("chunkinfos"sv); for (size_t Index = 0; Index < Count; Index++) @@ -4250,7 +4325,7 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId, const Oid Obj = Oid::FromHexString(ChunkId); - IoBuffer Chunk = FoundLog->FindChunk(Obj); + IoBuffer Chunk = FoundLog->FindChunk(Obj, nullptr); if (!Chunk) { return {HttpResponseCode::NotFound, {}}; @@ -4284,7 +4359,8 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, uint64_t Size, ZenContentType AcceptType, CompositeBuffer& OutChunk, - ZenContentType& OutContentType) + ZenContentType& OutContentType, + uint64_t* OptionalInOutModificationTag) { if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) { @@ -4293,41 +4369,19 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, const Oid Obj = Oid::FromHexString(ChunkId); - return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType); + return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType, OptionalInOutModificationTag); } -std::pair<HttpResponseCode, std::string> -ProjectStore::GetChunkRange(const std::string_view ProjectId, - const std::string_view OplogId, - Oid ChunkId, - uint64_t Offset, - uint64_t Size, - ZenContentType AcceptType, - CompositeBuffer& OutChunk, - ZenContentType& OutContentType) +static std::pair<HttpResponseCode, std::string> +ExtractRange(IoBuffer&& Chunk, + uint64_t Offset, + uint64_t Size, + ZenContentType AcceptType, + ZenContentType& OutContentType, + CompositeBuffer& OutChunk, + IoHash& OutRawHash, + uint64_t& OutRawSize) { - bool IsOffset = Offset != 0 || Size != ~(0ull); - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); - if (!FoundLog) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - IoBuffer Chunk = FoundLog->FindChunk(ChunkId); - if (!Chunk) - { - return {HttpResponseCode::NotFound, {}}; - } - OutContentType = Chunk.GetContentType(); if (OutContentType == ZenContentType::kCompressedBinary) @@ -4335,26 +4389,47 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, IoHash RawHash; uint64_t RawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), RawHash, RawSize); - ZEN_ASSERT(!Compressed.IsNull()); + if (!Compressed) + { + return {HttpResponseCode::InternalServerError, "malformed compressed buffer"}; + } + + const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == RawSize)); - if (IsOffset) + if (IsFullRange) + { + if (AcceptType == ZenContentType::kBinary) + { + OutChunk = Compressed.DecompressToComposite(); + OutContentType = ZenContentType::kBinary; + } + else + { + OutChunk = Compressed.GetCompressed(); + } + OutRawSize = 0; + } + else { if (Size == ~(0ull) || (Offset + Size) > RawSize) { - Size = RawSize - Offset; + if (Offset < RawSize) + { + Size = RawSize - Offset; + } + else + { + Size = 0; + } } if (Size == 0) { - return { - HttpResponseCode::NotFound, - fmt::format("Chunk request for range outside of chunk '{}/{}'. Request: Chunk: {}, Offset: {}, Size: {}, ChunkSize: {}", - ProjectId, - OplogId, - ChunkId, - Offset, - Size, - RawSize)}; + return {HttpResponseCode::NotFound, + fmt::format("Chunk request for range outside of compressed chunk. Offset: {}, Size: {}, ChunkSize: {}", + Offset, + Size, + RawSize)}; } if (AcceptType == ZenContentType::kBinary) @@ -4368,47 +4443,93 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, // The client will have to compensate for any offsets that do not land on an even block size multiple OutChunk = Compressed.GetRange(Offset, Size).GetCompressed(); } + OutRawSize = RawSize; + } + OutRawHash = RawHash; + } + else + { + const uint64_t ChunkSize = Chunk.GetSize(); + + const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == ChunkSize)); + if (IsFullRange) + { + OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk))); + OutRawSize = 0; } else { - if (AcceptType == ZenContentType::kBinary) + if (Size == ~(0ull) || (Offset + Size) > ChunkSize) { - OutChunk = Compressed.DecompressToComposite(); - OutContentType = ZenContentType::kBinary; + if (Offset < ChunkSize) + { + Size = ChunkSize - Offset; + } + else + { + Size = 0; + } } - else + + if (Size == 0) { - OutChunk = Compressed.GetCompressed(); + return {HttpResponseCode::NotFound, + fmt::format("Chunk request for range outside of chunk. Offset: {}, Size: {}, ChunkSize: {}", Offset, Size, Size)}; } + + OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size))); + OutRawSize = ChunkSize; } } - else if (IsOffset) + return {HttpResponseCode::OK, {}}; +} + +std::pair<HttpResponseCode, std::string> +ProjectStore::GetChunkRange(const std::string_view ProjectId, + const std::string_view OplogId, + Oid ChunkId, + uint64_t Offset, + uint64_t Size, + ZenContentType AcceptType, + CompositeBuffer& OutChunk, + ZenContentType& OutContentType, + uint64_t* OptionalInOutModificationTag) +{ + Ref<ProjectStore::Project> Project = OpenProject(ProjectId); + if (!Project) { - if (Size == ~(0ull) || (Offset + Size) > Chunk.GetSize()) - { - Size = Chunk.GetSize() - Offset; - } + return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)}; + } + Project->TouchProject(); - if (Size == 0) - { - return {HttpResponseCode::NotFound, - fmt::format("Chunk request for range outside of chunk '{}/{}'. Request: Chunk: {}, Offset: {}, Size: {}, ChunkSize: {}", - ProjectId, - OplogId, - ChunkId, - Offset, - Size, - Chunk.GetSize())}; - } + ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)}; + } + Project->TouchOplog(OplogId); - OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size))); + uint64_t OldTag = OptionalInOutModificationTag == nullptr ? 0 : *OptionalInOutModificationTag; + IoBuffer Chunk = FoundLog->FindChunk(ChunkId, OptionalInOutModificationTag); + if (!Chunk) + { + return {HttpResponseCode::NotFound, {}}; } - else + if (OptionalInOutModificationTag != nullptr && OldTag == *OptionalInOutModificationTag) { - OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk))); + return {HttpResponseCode::NotModified, {}}; } - return {HttpResponseCode::OK, {}}; + IoHash _; + uint64_t __; + std::pair<HttpResponseCode, std::string> Result = + ExtractRange(std::move(Chunk), Offset, Size, AcceptType, OutContentType, OutChunk, /*OutRawHash*/ _, /*OutRawSize*/ __); + if (Result.first != HttpResponseCode::OK) + { + return {Result.first, + fmt::format("Chunk request for chunk {} in {}/{} failed. Reason: '{}'", ChunkId, ProjectId, OplogId, Result.second)}; + } + return Result; } std::pair<HttpResponseCode, std::string> @@ -4416,7 +4537,8 @@ ProjectStore::GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, ZenContentType AcceptType, - IoBuffer& OutChunk) + IoBuffer& OutChunk, + uint64_t* OptionalInOutModificationTag) { Ref<ProjectStore::Project> Project = OpenProject(ProjectId); if (!Project) @@ -4445,6 +4567,16 @@ ProjectStore::GetChunk(const std::string_view ProjectId, return {HttpResponseCode::NotFound, fmt::format("chunk - '{}' MISSING", Cid)}; } + if (OptionalInOutModificationTag != nullptr) + { + uint64_t OldTag = *OptionalInOutModificationTag; + *OptionalInOutModificationTag = GetModificationTagFromRawHash(Hash); + if (*OptionalInOutModificationTag == OldTag) + { + return {HttpResponseCode::NotModified, {}}; + } + } + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); @@ -4504,6 +4636,354 @@ ProjectStore::PutChunk(const std::string_view ProjectId, } std::pair<HttpResponseCode, std::string> +ProjectStore::GetChunks(const std::string_view ProjectId, + const std::string_view OplogId, + const CbObject& RequestObject, + CbPackage& OutResponsePackage) +{ + ZEN_TRACE_CPU("Store::GetChunks"); + + using namespace std::literals; + + Ref<ProjectStore::Project> Project = OpenProject(ProjectId); + if (!Project) + { + return {HttpResponseCode::NotFound, fmt::format("getchunks rpc request for unknown project '{}'", ProjectId)}; + } + Project->TouchProject(); + + ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return {HttpResponseCode::NotFound, fmt::format("getchunks rpc request for unknown oplog '{}/{}'", ProjectId, OplogId)}; + } + Project->TouchOplog(OplogId); + + if (RequestObject["chunks"sv].IsArray()) + { + // Legacy full chunks only by rawhash + + CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView(); + + CbObjectWriter ResponseWriter; + ResponseWriter.BeginArray("chunks"sv); + for (CbFieldView FieldView : ChunksArray) + { + IoHash RawHash = FieldView.AsHash(); + IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash); + if (ChunkBuffer) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer)); + if (Compressed) + { + ResponseWriter.AddHash(RawHash); + OutResponsePackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash)); + } + else + { + ZEN_WARN("oplog '{}/{}': invalid compressed binary in cas store for {}", ProjectId, OplogId, RawHash); + } + } + } + ResponseWriter.EndArray(); + OutResponsePackage.SetObject(ResponseWriter.Save()); + return {HttpResponseCode::OK, {}}; + } + else if (auto RequestFieldView = RequestObject["Request"sv]; RequestFieldView.IsObject()) + { + CbObjectView RequestView = RequestFieldView.AsObjectView(); + bool SkipData = RequestView["SkipData"].AsBool(false); + CbArrayView ChunksArray = RequestView["Chunks"sv].AsArrayView(); + + struct Request + { + struct InputData + { + uint64_t Offset = 0; + uint64_t Size = (uint64_t)-1; + std::variant<IoHash, Oid> Id; + std::optional<uint64_t> ModTag; + } Input; + struct OutputData + { + bool Exists = false; + IoBuffer ChunkBuffer; + uint64_t ModTag = 0; + } Output; + }; + + std::vector<Request> Requests; + size_t RequestCount = ChunksArray.Num(); + if (RequestCount > 0) + { + Requests.reserve(RequestCount); + std::vector<IoHash> ChunkRawHashes; + std::vector<size_t> ChunkRawHashesRequestIndex; + std::vector<Oid> ChunkIds; + std::vector<size_t> ChunkIdsRequestIndex; + bool DoBatch = RequestCount > 1; + if (DoBatch) + { + ChunkRawHashes.reserve(RequestCount); + ChunkRawHashesRequestIndex.reserve(RequestCount); + ChunkIds.reserve(RequestCount); + ChunkIdsRequestIndex.reserve(RequestCount); + } + for (CbFieldView FieldView : ChunksArray) + { + CbObjectView ChunkObject = FieldView.AsObjectView(); + Request ChunkRequest = { + .Input{.Offset = ChunkObject["Offset"sv].AsUInt64(0), .Size = ChunkObject["Size"sv].AsUInt64((uint64_t)-1)}}; + if (CbFieldView InputModificationTagView = ChunkObject.FindView("ModTag"); InputModificationTagView.IsInteger()) + { + ChunkRequest.Input.ModTag = InputModificationTagView.AsUInt64(); + } + if (CbFieldView RawHashView = ChunkObject.FindView("RawHash"sv); RawHashView.IsHash()) + { + const IoHash ChunkHash = RawHashView.AsHash(); + ChunkRequest.Input.Id = ChunkHash; + if (DoBatch) + { + ChunkRawHashes.push_back(ChunkHash); + ChunkRawHashesRequestIndex.push_back(Requests.size()); + } + } + else if (CbFieldView IdView = ChunkObject.FindView("Oid"sv); IdView.IsObjectId()) + { + const Oid ChunkId = IdView.AsObjectId(); + ChunkRequest.Input.Id = ChunkId; + if (DoBatch) + { + ChunkIds.push_back(ChunkId); + ChunkIdsRequestIndex.push_back(Requests.size()); + } + } + else + { + return {HttpResponseCode::BadRequest, + fmt::format("oplog '{}/{}': malformed getchunks rpc request object, chunk request has no identifier", + ProjectId, + OplogId)}; + } + Requests.emplace_back(std::move(ChunkRequest)); + } + + if (DoBatch) + { + WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); + if (!ChunkRawHashes.empty()) + { + FoundLog->IterateChunks( + ChunkRawHashes, + true, + [&](size_t Index, const IoBuffer& Payload, uint64_t ModTag) -> bool { + if (Payload) + { + size_t RequestIndex = ChunkRawHashesRequestIndex[Index]; + Requests[RequestIndex].Output.Exists = true; + if (!SkipData) + { + Requests[RequestIndex].Output.ChunkBuffer = Payload; + Requests[RequestIndex].Output.ChunkBuffer.MakeOwned(); + } + Requests[RequestIndex].Output.ModTag = ModTag; + } + return true; + }, + &WorkerPool, + 8u * 1024); + } + if (!ChunkIdsRequestIndex.empty()) + { + FoundLog->IterateChunks( + ChunkIds, + true, + [&](size_t Index, const IoBuffer& Payload, uint64_t ModTag) -> bool { + if (Payload) + { + size_t RequestIndex = ChunkIdsRequestIndex[Index]; + Requests[RequestIndex].Output.Exists = true; + if (!SkipData) + { + Requests[RequestIndex].Output.ChunkBuffer = Payload; + Requests[RequestIndex].Output.ChunkBuffer.MakeOwned(); + } + Requests[RequestIndex].Output.ModTag = ModTag; + } + return true; + }, + &WorkerPool, + 8u * 1024); + } + } + else + { + Request& ChunkRequest = Requests.front(); + if (ChunkRequest.Input.Id.index() == 0) + { + const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id); + IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash); + if (Payload) + { + ChunkRequest.Output.Exists = true; + ChunkRequest.Output.ModTag = GetModificationTagFromRawHash(ChunkHash); + if (!SkipData) + { + ChunkRequest.Output.ChunkBuffer = Payload; + } + } + } + else + { + const Oid& ChunkId = std::get<Oid>(ChunkRequest.Input.Id); + uint64_t ModTag = 0; + IoBuffer Payload = FoundLog->FindChunk(ChunkId, &ModTag); + if (Payload) + { + ChunkRequest.Output.Exists = true; + ChunkRequest.Output.ModTag = ModTag; + if (!SkipData) + { + ChunkRequest.Output.ChunkBuffer = Payload; + } + } + } + } + } + + CbObjectWriter ResponseWriter(32 + Requests.size() * 64u); + ResponseWriter.BeginArray("Chunks"sv); + { + for (Request& ChunkRequest : Requests) + { + if (ChunkRequest.Output.Exists) + { + ResponseWriter.BeginObject(); + { + if (ChunkRequest.Input.Id.index() == 0) + { + const IoHash& RawHash = std::get<IoHash>(ChunkRequest.Input.Id); + ResponseWriter.AddHash("Id", RawHash); + } + else + { + const Oid& Id = std::get<Oid>(ChunkRequest.Input.Id); + ResponseWriter.AddObjectId("Id", Id); + } + if (!ChunkRequest.Input.ModTag.has_value() || ChunkRequest.Input.ModTag.value() != ChunkRequest.Output.ModTag) + { + ResponseWriter.AddInteger("ModTag", ChunkRequest.Output.ModTag); + if (!SkipData) + { + CompositeBuffer ChunkRange; + ZenContentType ContentType; + IoHash FullChunkRawHash; + uint64_t FullChunkSize = 0; + auto ExtractRangeResult = ExtractRange(std::move(ChunkRequest.Output.ChunkBuffer), + ChunkRequest.Input.Offset, + ChunkRequest.Input.Size, + ZenContentType::kCompressedBinary, + ContentType, + ChunkRange, + FullChunkRawHash, + FullChunkSize); + if (ExtractRangeResult.first == HttpResponseCode::OK) + { + if (ContentType == ZenContentType::kCompressedBinary) + { + ZEN_ASSERT(FullChunkRawHash != IoHash::Zero); + CompressedBuffer CompressedValue = + CompressedBuffer::FromCompressedNoValidate(std::move(ChunkRange)); + ZEN_ASSERT(CompressedValue); + + if (FullChunkSize != 0) + { + // This really could use some thought so we don't send the same data if we get a request for + // multiple ranges from the same chunk block + + uint64_t FragmentRawOffset = 0; + OodleCompressor Compressor; + OodleCompressionLevel CompressionLevel; + uint64_t BlockSize = 0; + if (CompressedValue.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize)) + { + if (BlockSize > 0) + { + FragmentRawOffset = (ChunkRequest.Input.Offset / BlockSize) * BlockSize; + } + else + { + FragmentRawOffset = ChunkRequest.Input.Offset; + } + uint64_t FragmentRawLength = CompressedValue.DecodeRawSize(); + + IoHashStream FragmentHashStream; + FragmentHashStream.Append(FullChunkRawHash.Hash, sizeof(FullChunkRawHash.Hash)); + FragmentHashStream.Append(&FragmentRawOffset, sizeof(FragmentRawOffset)); + FragmentHashStream.Append(&FragmentRawLength, sizeof(FragmentRawLength)); + IoHash FragmentHash = FragmentHashStream.GetHash(); + + ResponseWriter.AddHash("FragmentHash", FragmentHash); + ResponseWriter.AddInteger("FragmentOffset", FragmentRawOffset); + ResponseWriter.AddInteger("RawSize", FullChunkSize); + OutResponsePackage.AddAttachment(CbAttachment(CompressedValue, FragmentHash)); + } + else + { + std::string ErrorString = + "Failed to get compression parameters from partial compressed buffer"; + ResponseWriter.AddString("Error", ErrorString); + ZEN_WARN("oplog '{}/{}': {}", ProjectId, OplogId, ErrorString); + } + } + else + { + ResponseWriter.AddHash("RawHash"sv, FullChunkRawHash); + OutResponsePackage.AddAttachment(CbAttachment(std::move(CompressedValue), FullChunkRawHash)); + } + } + else + { + IoHashStream HashStream; + ZEN_ASSERT(ChunkRequest.Input.Id.index() == 1); + const Oid& Id = std::get<Oid>(ChunkRequest.Input.Id); + HashStream.Append(Id.OidBits, sizeof(Id.OidBits)); + HashStream.Append(&ChunkRequest.Input.Offset, sizeof(ChunkRequest.Input.Offset)); + HashStream.Append(&ChunkRequest.Input.Size, sizeof(ChunkRequest.Input.Size)); + IoHash Hash = HashStream.GetHash(); + + ResponseWriter.AddHash("Hash"sv, Hash); + if (FullChunkSize != 0) + { + ResponseWriter.AddInteger("Size", FullChunkSize); + } + OutResponsePackage.AddAttachment(CbAttachment(std::move(ChunkRange), Hash)); + } + } + else + { + std::string ErrorString = fmt::format("Failed fetchiong chunk range ({})", ExtractRangeResult.second); + ResponseWriter.AddString("Error", ErrorString); + ZEN_WARN("oplog '{}/{}': {}", ProjectId, OplogId, ErrorString); + } + } + } + } + ResponseWriter.EndObject(); + } + } + } + ResponseWriter.EndArray(); + OutResponsePackage.SetObject(ResponseWriter.Save()); + return {HttpResponseCode::OK, {}}; + } + else + { + return {HttpResponseCode::BadRequest, fmt::format("oplog '{}/{}': malformed getchunks rpc request object", ProjectId, OplogId)}; + } +} + +std::pair<HttpResponseCode, std::string> ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_view OplogId, IoBuffer&& Payload, CbObject& OutResponse) { ZEN_TRACE_CPU("Store::WriteOplog"); @@ -4570,7 +5050,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie return ConvertResult(RemoteResult); } - CbObjectWriter Cbo; + CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1); Cbo.BeginArray("need"); { for (const IoHash& Hash : Attachments) @@ -4767,34 +5247,34 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, else if (Method == "getchunks"sv) { ZEN_TRACE_CPU("Store::Rpc::getchunks"); - CbPackage ResponsePackage; + + RpcAcceptOptions AcceptFlags = static_cast<RpcAcceptOptions>(Cb["AcceptFlags"sv].AsUInt16(0u)); + // TODO: Move OpenProccessCache to a more general place + // int32_t TargetProcessId = Cb["Pid"sv].AsInt32(0); + + CbPackage ResponsePackage; + std::pair<HttpResponseCode, std::string> Result = GetChunks(ProjectId, OplogId, Cb, ResponsePackage); + if (Result.first == HttpResponseCode::OK) { - CbArrayView ChunksArray = Cb["chunks"sv].AsArrayView(); - CbObjectWriter ResponseWriter; - ResponseWriter.BeginArray("chunks"sv); - for (CbFieldView FieldView : ChunksArray) + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) { - IoHash RawHash = FieldView.AsHash(); - IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash); - if (ChunkBuffer) + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer)); - if (Compressed) - { - ResponseWriter.AddHash(RawHash); - ResponsePackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash)); - } - else - { - ZEN_WARN("oplog '{}/{}': invalid compressed binary in cas store for {}", ProjectId, OplogId, RawHash); - } + Flags |= FormatFlags::kDenyPartialLocalReferences; } + // TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(Context.SessionId, TargetPid); } - ResponseWriter.EndArray(); - ResponsePackage.SetObject(ResponseWriter.Save()); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage, Flags, TargetProcessHandle); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + } + else + { + HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } - CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage, FormatFlags::kDefault); - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); return true; } else if (Method == "putchunks"sv) @@ -4927,7 +5407,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, ++TotalFiles; // Rewrite file array entry with new data reference - CbObjectWriter Writer; + CbObjectWriter Writer(View.GetSize()); RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { if (Field.GetName() == "data"sv) { @@ -6012,7 +6492,7 @@ namespace testutils { return OidStringBuilder.ToString(); } - CbPackage CreateOplogPackage(const Oid& Id, const std::span<const std::pair<Oid, CompressedBuffer>>& Attachments) + CbPackage CreateBulkDataOplogPackage(const Oid& Id, const std::span<const std::pair<Oid, CompressedBuffer>>& Attachments) { CbPackage Package; CbObjectWriter Object; @@ -6038,16 +6518,43 @@ namespace testutils { return Package; }; + CbPackage CreateFilesOplogPackage(const Oid& Id, + const std::filesystem::path ProjectRootDir, + const std::span<const std::pair<Oid, std::filesystem::path>>& Attachments) + { + CbPackage Package; + CbObjectWriter Object; + Object << "key"sv << OidAsString(Id); + if (!Attachments.empty()) + { + Object.BeginArray("files"); + for (const auto& Attachment : Attachments) + { + std::filesystem::path ServerPath = std::filesystem::relative(Attachment.second, ProjectRootDir); + std::filesystem::path ClientPath = ServerPath; // dummy + Object.BeginObject(); + Object << "id"sv << Attachment.first; + Object << "serverpath"sv << ServerPath.string(); + Object << "clientpath"sv << ClientPath.string(); + Object.EndObject(); + } + Object.EndArray(); + } + Package.SetObject(Object.Save()); + return Package; + }; + std::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments( const std::span<const size_t>& Sizes, - OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast) + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0) { std::vector<std::pair<Oid, CompressedBuffer>> Result; Result.reserve(Sizes.size()); for (size_t Size : Sizes) { CompressedBuffer Compressed = - CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(Size)), OodleCompressor::Mermaid, CompressionLevel); + CompressedBuffer::Compress(SharedBuffer(CreateSemiRandomBlob(Size)), OodleCompressor::Mermaid, CompressionLevel, BlockSize); Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed)); } return Result; @@ -6069,6 +6576,63 @@ namespace testutils { return 0; } + template<typename ChunkType> + CbObject BuildChunksRequest(bool SkipData, + std::string_view IdName, + const std::vector<ChunkType>& Chunks, + const std::vector<std::pair<size_t, size_t>>& Ranges, + const std::vector<uint64_t>& ModTags) + { + CbObjectWriter Request; + Request.BeginObject("Request"sv); + { + if (SkipData) + { + Request.AddBool("SkipData"sv, true); + } + if (!Chunks.empty()) + { + Request.BeginArray("Chunks"); + for (size_t Index = 0; Index < Chunks.size(); Index++) + { + Request.BeginObject(); + { + Request << IdName << Chunks[Index]; + if (!ModTags.empty()) + { + Request << "ModTag" << ModTags[Index]; + } + if (!Ranges.empty()) + { + Request << "Offset" << Ranges[Index].first; + Request << "Size" << Ranges[Index].second; + } + } + Request.EndObject(); + } + Request.EndArray(); + } + } + Request.EndObject(); + return Request.Save(); + }; + + CbObject BuildChunksRequest(bool SkipData, + const std::vector<Oid>& Chunks, + const std::vector<std::pair<size_t, size_t>>& Ranges, + const std::vector<uint64_t>& ModTags) + { + return BuildChunksRequest<Oid>(SkipData, "Oid", Chunks, Ranges, ModTags); + } + + CbObject BuildChunksRequest(bool SkipData, + const std::vector<IoHash>& Chunks, + const std::vector<std::pair<size_t, size_t>>& Ranges, + const std::vector<uint64_t>& ModTags) + { + return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags); + } + } // namespace testutils TEST_CASE("project.store.create") @@ -6191,13 +6755,14 @@ TEST_CASE_TEMPLATE("project.store.export", ProjectStore::Oplog* Oplog = Project->NewOplog("oplog1", {}); CHECK(Oplog != nullptr); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); Oplog->AppendNewOplogEntry( - CreateOplogPackage(Oid::NewOid(), - CreateAttachments(std::initializer_list<size_t>{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None))); + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage( + Oid::NewOid(), + CreateAttachments(std::initializer_list<size_t>{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None))); FileRemoteStoreOptions Options = { RemoteStoreOptions{.MaxBlockSize = 64u * 1024, .MaxChunkEmbedSize = 32 * 1024u, .ChunkFileSizeLimit = 64u * 1024u}, @@ -6325,10 +6890,11 @@ TEST_CASE("project.store.gc") ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1", Project1OplogPath); CHECK(Oplog != nullptr); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); + Oplog->AppendNewOplogEntry( + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); } { @@ -6342,21 +6908,23 @@ TEST_CASE("project.store.gc") ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog2", Project2Oplog1Path); CHECK(Oplog != nullptr); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177}))); + Oplog->AppendNewOplogEntry( + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96}))); Oplog->AppendNewOplogEntry( - CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221}))); + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221}))); } { ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog3", Project2Oplog2Path); CHECK(Oplog != nullptr); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{137}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{137}))); Oplog->AppendNewOplogEntry( - CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9723, 683, 594, 98}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{531, 271}))); + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9723, 683, 594, 98}))); + Oplog->AppendNewOplogEntry( + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{531, 271}))); } } @@ -6507,7 +7075,7 @@ TEST_CASE("project.store.gc.prep") Project1RootDir.string(), Project1FilePath.string())); ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments)); } { Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); @@ -6538,7 +7106,7 @@ TEST_CASE("project.store.gc.prep") // Make sure the chunks are stored but not the referencing op Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments)); Project1->DeleteOplog("oplog1"sv); } { @@ -6585,7 +7153,7 @@ TEST_CASE("project.store.gc.prep") { Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); ProjectStore::Oplog* Oplog = Project1->OpenOplog("oplog1"sv, true, true); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments)); Oplog->RemovePendingChunkReferences(OpChunkHashes); CHECK(Oplog->GetPendingChunkReferencesLocked().size() == 0); } @@ -6619,7 +7187,7 @@ TEST_CASE("project.store.gc.prep") // Make sure the chunks are stored but not the referencing op Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments)); Project1->DeleteOplog("oplog1"sv); } @@ -6674,6 +7242,927 @@ TEST_CASE("project.store.gc.prep") } } +TEST_CASE("project.store.rpc.getchunks") +{ + using namespace std::literals; + using namespace testutils; + + ScopedTemporaryDirectory TempDir; + + auto JobQueue = MakeJobQueue(1, ""sv); + GcManager Gc; + CidStore CidStore(Gc); + CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; + CidStore.Initialize(CidConfig); + + std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{}); + std::filesystem::path RootDir = TempDir.Path() / "root"sv; + std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; + + std::filesystem::path Project1RootDir = TempDir.Path() / "game1"sv; + std::filesystem::path Project1FilePath = TempDir.Path() / "game1"sv / "game.uproject"sv; + { + CreateDirectories(Project1FilePath.parent_path()); + BasicFile ProjectFile; + ProjectFile.Open(Project1FilePath, BasicFile::Mode::kTruncate); + } + + std::vector<Oid> OpIds; + OpIds.insert(OpIds.end(), {Oid::NewOid(), Oid::NewOid(), Oid::NewOid(), Oid::NewOid()}); + std::unordered_map<Oid, std::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; + Oid FilesOpId = Oid::NewOid(); + std::vector<std::pair<Oid, std::filesystem::path>> FilesOpIdAttachments; + { + Ref<ProjectStore::Project> Project1(ProjectStore.NewProject(BasePath / "proj1"sv, + "proj1"sv, + RootDir.string(), + EngineRootDir.string(), + Project1RootDir.string(), + Project1FilePath.string())); + ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, {}); + CHECK(Oplog != nullptr); + Attachments[OpIds[0]] = {}; + Attachments[OpIds[1]] = CreateAttachments(std::initializer_list<size_t>{77}); + Attachments[OpIds[2]] = + CreateAttachments(std::initializer_list<size_t>{200 * 1024, 314 * 1024, 690, 99}, OodleCompressionLevel::VeryFast, 128 * 1024); + Attachments[OpIds[3]] = CreateAttachments(std::initializer_list<size_t>{55, 122}); + for (auto It : Attachments) + { + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(It.first, It.second)); + } + + std::filesystem::path UncompressedFilePath = RootDir / "content" / "uncompressed_file"; + CreateDirectories(UncompressedFilePath.parent_path()); + IoBuffer FileBlob = CreateRandomBlob(81823 * 2); + WriteFile(UncompressedFilePath, FileBlob); + FilesOpIdAttachments.push_back({Oid::NewOid(), UncompressedFilePath}); + Oplog->AppendNewOplogEntry(CreateFilesOplogPackage(FilesOpId, RootDir, FilesOpIdAttachments)); + } + + // Invalid request + { + CbObjectWriter Request; + Request.BeginObject("WrongName"sv); + Request.EndObject(); + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, "oplog1"sv, Request.Save(), Response); + CHECK_EQ(HttpResponseCode::BadRequest, Result.first); + } + + // Empty request + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, std::vector<IoHash>{}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + // Single non-existing chunk by RawHash + IoHash NotFoundIoHash = IoHash::Max; + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {NotFoundIoHash}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {NotFoundIoHash}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + // Single non-existing chunk by Id + Oid NotFoundId = Oid::NewOid(); + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {NotFoundId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, "oplog1"sv, testutils::BuildChunksRequest(/*SkipData*/ true, {NotFoundId}, {}, {}), Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(0, Chunks.Num()); + } + // Single existing chunk by RawHash + { + // Fresh fetch + IoHash FirstAttachmentHash = Attachments[OpIds[2]][1].second.DecodeRawHash(); + uint64_t ResponseModTag = 0; + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fetch with matching ModTag + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fetch with mismatching ModTag + { + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fresh modtime query + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with matching ModTag + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with mismatching ModTag + { + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + IoHash Id = Chunk["Id"].AsHash(); + CHECK_EQ(FirstAttachmentHash, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + // Single existing CID chunk by Id + { + Oid FirstAttachmentId = Attachments[OpIds[2]][1].first; + uint64_t ResponseModTag = 0; + { + // Full chunk request + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Partial chunk request + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {{130 * 1024, 8100}}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["FragmentHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + std::uint64_t FragmentStart = Chunk["FragmentOffset"].AsUInt64(); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK(FragmentStart <= 130 * 1024); + CHECK(FragmentStart + Buffer.DecodeRawSize() >= 130 * 1024 + 8100); + auto ResponseDecompressedBuffer = Buffer.Decompress(130 * 1024 - FragmentStart, 8100); + auto ExpectedDecompressedBuffer = Attachments[OpIds[2]][1].second.Decompress(130 * 1024, 8100); + CHECK(ResponseDecompressedBuffer.AsIoBuffer().GetView().EqualBytes(ExpectedDecompressedBuffer.AsIoBuffer().GetView())); + CHECK_EQ(Chunk["RawSize"sv].AsUInt64(), Attachments[OpIds[2]][1].second.DecodeRawSize()); + CHECK(!Chunk.FindView("Size")); + } + { + // Fetch with matching ModTag + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Fetch with mismatching ModTag + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag3 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag3); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fresh modtime query + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with matching ModTag + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with mismatching ModTag + { + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + + // Single existing file chunk by Id + { + Oid FirstAttachmentId = FilesOpIdAttachments[0].first; + uint64_t ResponseModTag = 0; + { + // Full chunk request + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + IoHash AttachmentHash = Chunk["Hash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompositeBuffer Buffer = Attachment->AsCompositeBinary(); + CHECK_EQ(IoHash::HashBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten()), IoHash::HashBuffer(Buffer)); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Partial chunk request + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {{81823, 5434}}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + ResponseModTag = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTag); + + IoHash AttachmentHash = Chunk["Hash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompositeBuffer Buffer = Attachment->AsCompositeBinary(); + CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)), + IoHash::HashBuffer(Buffer)); + CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second)); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Fetch with matching ModTag + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("Hash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + { + // Fetch with mismatching ModTag + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(1, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag3 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag3); + IoHash AttachmentHash = Chunk["Hash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompositeBuffer Buffer = Attachment->AsCompositeBinary(); + CHECK_EQ(IoHash::HashBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten()), IoHash::HashBuffer(Buffer)); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Fresh modtime query + { + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with matching ModTag + { + CbPackage Response; + auto Result = + ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {ResponseModTag}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("Hash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + // Modtime query with mismatching ModTag + { + CbPackage Response; + auto Result = ProjectStore.GetChunks( + "proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}), + Response); + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(1, Chunks.Num()); + CbObjectView Chunk = (*begin(Chunks)).AsObjectView(); + + Oid Id = Chunk["Id"].AsObjectId(); + CHECK_EQ(FirstAttachmentId, Id); + uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64(); + CHECK_EQ(ResponseModTag, ResponseModTag2); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + + // Multi RawHash Request + { + std::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second, + Attachments[OpIds[2]][0].second, + Attachments[OpIds[2]][1].second}; + std::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(), + AttachmentBuffers[1].DecodeRawHash(), + AttachmentBuffers[2].DecodeRawHash()}; + std::vector<uint64_t> ResponseModTags(3, 0); + { + // Fresh fetch + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, AttachmentHashes, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(3, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + ResponseModTags[Index] = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTags[Index]); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(AttachmentBuffers[Index].GetCompressed().Flatten().GetView().EqualBytes(Buffer.GetCompressed().Flatten().GetView())); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Fetch with matching ModTag + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, AttachmentHashes, {}, ResponseModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Fresh modtime query + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + CHECK_EQ(ResponseModTags[Index], Chunk["ModTag"].AsUInt64()); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Modtime query with matching ModTags + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, ResponseModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Modtime query with mismatching ModTags + std::vector<uint64_t> MismatchingModTags(ResponseModTags); + for (uint64_t& Tag : MismatchingModTags) + { + Tag++; + } + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, MismatchingModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + IoHash Id = Chunk["Id"].AsHash(); + + auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id); + CHECK(It != AttachmentHashes.end()); + ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It); + CHECK_EQ(AttachmentHashes[Index], Id); + CHECK(Chunk["ModTag"].AsUInt64() == ResponseModTags[Index]); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + } + // Multi Id Request + { + std::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second, + Attachments[OpIds[2]][0].second, + Attachments[OpIds[2]][1].second}; + std::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(), + AttachmentBuffers[1].DecodeRawHash(), + AttachmentBuffers[2].DecodeRawHash()}; + std::vector<Oid> AttachedIds{Attachments[OpIds[1]][0].first, Attachments[OpIds[2]][0].first, Attachments[OpIds[2]][1].first}; + std::vector<uint64_t> ResponseModTags(3, 0); + { + // Fresh fetch + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, AttachedIds, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(3, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + ResponseModTags[Index] = Chunk["ModTag"].AsUInt64(); + CHECK_NE(0, ResponseModTags[Index]); + IoHash AttachmentHash = Chunk["RawHash"].AsHash(); + const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash); + CHECK_NE(nullptr, Attachment); + CompressedBuffer Buffer = Attachment->AsCompressedBinary(); + CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash()); + CHECK(AttachmentBuffers[Index].GetCompressed().Flatten().GetView().EqualBytes(Buffer.GetCompressed().Flatten().GetView())); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Fetch with matching ModTag + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ false, AttachedIds, {}, ResponseModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Fresh modtime query + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, {}), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + CHECK_EQ(ResponseModTags[Index], Chunk["ModTag"].AsUInt64()); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Modtime query with matching ModTags + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, ResponseModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + CHECK(!Chunk.FindView("ModTag")); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + { + // Modtime query with mismatching ModTags + std::vector<uint64_t> MismatchingModTags(ResponseModTags); + for (uint64_t& Tag : MismatchingModTags) + { + Tag++; + } + CbPackage Response; + auto Result = ProjectStore.GetChunks("proj1"sv, + "oplog1"sv, + testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, MismatchingModTags), + Response); + + CHECK_EQ(HttpResponseCode::OK, Result.first); + CHECK_EQ(0, Response.GetAttachments().size()); + CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView(); + CHECK_EQ(3, Chunks.Num()); + for (CbFieldView ChunkView : Chunks) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + Oid Id = Chunk["Id"].AsObjectId(); + + auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id); + CHECK(It != AttachedIds.end()); + ptrdiff_t Index = std::distance(AttachedIds.begin(), It); + CHECK_EQ(AttachedIds[Index], Id); + CHECK(Chunk["ModTag"].AsUInt64() == ResponseModTags[Index]); + CHECK(!Chunk.FindView("RawHash")); + CHECK(!Chunk.FindView("Size")); + CHECK(!Chunk.FindView("RawSize")); + } + } + } +} + TEST_CASE("project.store.partial.read") { using namespace std::literals; @@ -6718,63 +8207,119 @@ TEST_CASE("project.store.partial.read") Attachments[OpIds[3]] = CreateAttachments(std::initializer_list<size_t>{55, 122}); for (auto It : Attachments) { - Oplog->AppendNewOplogEntry(CreateOplogPackage(It.first, It.second)); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(It.first, It.second)); } } { + uint64_t ModificationTag = 0; IoBuffer Chunk; CHECK(ProjectStore .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), HttpContentType::kCompressedBinary, - Chunk) + Chunk, + &ModificationTag) .first == HttpResponseCode::OK); IoHash RawHash; uint64_t RawSize; CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize); CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize()); - } - CompositeBuffer ChunkResult; - HttpContentType ContentType; - CHECK(ProjectStore - .GetChunkRange("proj1"sv, - "oplog1"sv, - OidAsString(Attachments[OpIds[2]][1].first), - 0, - ~0ull, - HttpContentType::kCompressedBinary, - ChunkResult, - ContentType) - .first == HttpResponseCode::OK); - CHECK(ChunkResult); - CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() == - Attachments[OpIds[2]][1].second.DecodeRawSize()); - - CompositeBuffer PartialChunkResult; - CHECK(ProjectStore - .GetChunkRange("proj1"sv, - "oplog1"sv, - OidAsString(Attachments[OpIds[2]][1].first), - 5, - 1773, - HttpContentType::kCompressedBinary, - PartialChunkResult, - ContentType) - .first == HttpResponseCode::OK); - CHECK(PartialChunkResult); - IoHash PartialRawHash; - uint64_t PartialRawSize; - CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize); - CHECK(PartialRawSize >= 1773); - - uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5); - SharedBuffer PartialDecompressed = PartialCompressedResult.Decompress(RawOffsetInPartialCompressed); - SharedBuffer FullDecompressed = Attachments[OpIds[2]][1].second.Decompress(); - const uint8_t* FullDataPtr = &(reinterpret_cast<const uint8_t*>(FullDecompressed.GetView().GetData())[5]); - const uint8_t* PartialDataPtr = reinterpret_cast<const uint8_t*>(PartialDecompressed.GetView().GetData()); - CHECK(FullDataPtr[0] == PartialDataPtr[0]); + CHECK(ModificationTag != 0); + CHECK(ProjectStore + .GetChunk("proj1"sv, + "oplog1"sv, + Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), + HttpContentType::kCompressedBinary, + Chunk, + &ModificationTag) + .first == HttpResponseCode::NotModified); + } + + { + uint64_t FullChunkModificationTag = 0; + { + CompositeBuffer ChunkResult; + HttpContentType ContentType; + CHECK(ProjectStore + .GetChunkRange("proj1"sv, + "oplog1"sv, + OidAsString(Attachments[OpIds[2]][1].first), + 0, + ~0ull, + HttpContentType::kCompressedBinary, + ChunkResult, + ContentType, + &FullChunkModificationTag) + .first == HttpResponseCode::OK); + CHECK(ChunkResult); + CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() == + Attachments[OpIds[2]][1].second.DecodeRawSize()); + } + { + CompositeBuffer ChunkResult; + HttpContentType ContentType; + CHECK(ProjectStore + .GetChunkRange("proj1"sv, + "oplog1"sv, + OidAsString(Attachments[OpIds[2]][1].first), + 0, + ~0ull, + HttpContentType::kCompressedBinary, + ChunkResult, + ContentType, + &FullChunkModificationTag) + .first == HttpResponseCode::NotModified); + } + } + { + CompositeBuffer PartialChunkResult; + uint64_t PartialChunkModificationTag = 0; + { + CompositeBuffer ChunkResult; + HttpContentType ContentType; + CHECK(ProjectStore + .GetChunkRange("proj1"sv, + "oplog1"sv, + OidAsString(Attachments[OpIds[2]][1].first), + 5, + 1773, + HttpContentType::kCompressedBinary, + PartialChunkResult, + ContentType, + &PartialChunkModificationTag) + .first == HttpResponseCode::OK); + CHECK(PartialChunkResult); + IoHash PartialRawHash; + uint64_t PartialRawSize; + CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize); + CHECK(PartialRawSize >= 1773); + + uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5); + SharedBuffer PartialDecompressed = PartialCompressedResult.Decompress(RawOffsetInPartialCompressed); + SharedBuffer FullDecompressed = Attachments[OpIds[2]][1].second.Decompress(); + const uint8_t* FullDataPtr = &(reinterpret_cast<const uint8_t*>(FullDecompressed.GetView().GetData())[5]); + const uint8_t* PartialDataPtr = reinterpret_cast<const uint8_t*>(PartialDecompressed.GetView().GetData()); + CHECK(FullDataPtr[0] == PartialDataPtr[0]); + } + + { + CompositeBuffer ChunkResult; + HttpContentType ContentType; + CHECK(ProjectStore + .GetChunkRange("proj1"sv, + "oplog1"sv, + OidAsString(Attachments[OpIds[2]][1].first), + 5, + 1773, + HttpContentType::kCompressedBinary, + PartialChunkResult, + ContentType, + &PartialChunkModificationTag) + .first == HttpResponseCode::NotModified); + } + } } TEST_CASE("project.store.block") @@ -6851,7 +8396,7 @@ TEST_CASE("project.store.iterateoplog") TestOidData TestOids[NumTestOids]; for (const TestOidData& TestOid : TestOids) { - Oplog->AppendNewOplogEntry(CreateOplogPackage(TestOid.KeyAsOidNotOplogId, {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(TestOid.KeyAsOidNotOplogId, {})); } int Count = 0; diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 860f2c17d..4a33a367c 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -114,16 +114,18 @@ public: std::optional<CbObject> GetOpByIndex(uint32_t Index); std::optional<uint32_t> GetOpIndexByKey(const Oid& Key); - IoBuffer FindChunk(const Oid& ChunkId); + IoBuffer FindChunk(const Oid& ChunkId, uint64_t* OptOutModificationTag); IoBuffer GetChunkByRawHash(const IoHash& RawHash); - bool IterateChunks(std::span<IoHash> RawHashes, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit); - bool IterateChunks(std::span<Oid> ChunkIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit); + bool IterateChunks(std::span<IoHash> RawHashes, + bool IncludeModTag, + const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit); + bool IterateChunks(std::span<Oid> ChunkIds, + bool IncludeModTag, + const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit); inline static const uint32_t kInvalidOp = ~0u; /** Persist a new oplog entry @@ -427,7 +429,8 @@ public: uint64_t Size, ZenContentType AcceptType, CompositeBuffer& OutChunk, - ZenContentType& OutContentType); + ZenContentType& OutContentType, + uint64_t* OptionalInOutModificationTag); std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view ChunkId, @@ -435,12 +438,14 @@ public: uint64_t Size, ZenContentType AcceptType, CompositeBuffer& OutChunk, - ZenContentType& OutContentType); + ZenContentType& OutContentType, + uint64_t* OptionalInOutModificationTag); std::pair<HttpResponseCode, std::string> GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, ZenContentType AcceptType, - IoBuffer& OutChunk); + IoBuffer& OutChunk, + uint64_t* OptionalInOutModificationTag); std::pair<HttpResponseCode, std::string> PutChunk(const std::string_view ProjectId, const std::string_view OplogId, @@ -458,6 +463,11 @@ public: const HttpServerRequest::QueryParams& Params, CbObject& OutResponse); + std::pair<HttpResponseCode, std::string> GetChunks(const std::string_view ProjectId, + const std::string_view OplogId, + const CbObject& RequestObject, + CbPackage& OutResponsePackage); + bool Rpc(HttpServerRequest& HttpReq, const std::string_view ProjectId, const std::string_view OplogId, diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp index c0082b746..6b05442b3 100644 --- a/src/zenserver/projectstore/zenremoteprojectstore.cpp +++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp @@ -152,14 +152,22 @@ public: { CbObjectWriter RequestWriter; RequestWriter.AddString("method"sv, "getchunks"sv); - RequestWriter.BeginArray("chunks"sv); + RequestWriter.BeginObject("Request"sv); { - for (const IoHash& RawHash : RawHashes) + RequestWriter.BeginArray("Chunks"sv); { - RequestWriter.AddHash(RawHash); + for (const IoHash& RawHash : RawHashes) + { + RequestWriter.BeginObject(); + { + RequestWriter.AddHash("RawHash", RawHash); + } + RequestWriter.EndObject(); + } } + RequestWriter.EndArray(); // "chunks" } - RequestWriter.EndArray(); // "chunks" + RequestWriter.EndObject(); Request = RequestWriter.Save(); } diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp index 6e14b7632..2bac6b756 100644 --- a/src/zenserver/vfs/vfsimpl.cpp +++ b/src/zenserver/vfs/vfsimpl.cpp @@ -47,7 +47,8 @@ VfsOplogDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t Byt ~0ull, ZenContentType::kCompressedBinary, /* out */ ChunkBuffer, - /* out */ ContentType); + /* out */ ContentType, + /* OptionalInOutModificationTag */ nullptr); if (Result.first == HttpResponseCode::OK) { diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 9aa4ff613..50af7246e 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -329,31 +329,33 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes, { ZEN_MEMSCOPE(GetCasContainerTag()); - const size_t ChunkCount = ChunkHashes.size(); - if (ChunkCount < 3) + const size_t ChunkCount = ChunkHashes.size(); + std::vector<size_t> FoundChunkIndexes; + std::vector<BlockStoreLocation> FoundChunkLocations; + FoundChunkIndexes.reserve(ChunkCount); + FoundChunkLocations.reserve(ChunkCount); { + RwLock::SharedLockScope _(m_LocationMapLock); for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ChunkIndex++) { - IoBuffer Chunk = FindChunk(ChunkHashes[ChunkIndex]); - if (!AsyncCallback(ChunkIndex, Chunk)) + if (auto KeyIt = m_LocationMap.find(ChunkHashes[ChunkIndex]); KeyIt != m_LocationMap.end()) { - return false; + FoundChunkIndexes.push_back(ChunkIndex); + FoundChunkLocations.push_back(m_Locations[KeyIt->second].Get(m_PayloadAlignment)); } } - return true; } - std::vector<size_t> FoundChunkIndexes; - std::vector<BlockStoreLocation> FoundChunkLocations; - FoundChunkIndexes.reserve(ChunkCount); - FoundChunkLocations.reserve(ChunkCount); - RwLock::SharedLockScope _(m_LocationMapLock); - for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ChunkIndex++) + if (FoundChunkLocations.size() < 3) { - if (auto KeyIt = m_LocationMap.find(ChunkHashes[ChunkIndex]); KeyIt != m_LocationMap.end()) + for (size_t ChunkIndex : FoundChunkIndexes) { - FoundChunkIndexes.push_back(ChunkIndex); - FoundChunkLocations.push_back(m_Locations[KeyIt->second].Get(m_PayloadAlignment)); + IoBuffer Chunk = m_BlockStore.TryGetChunk(FoundChunkLocations[ChunkIndex]); + if (!AsyncCallback(ChunkIndex, Chunk)) + { + return false; + } } + return true; } auto DoOneBlock = [&](std::span<const size_t> ChunkIndexes) { |