aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2024-12-05 14:21:12 +0100
committerGitHub Enterprise <[email protected]>2024-12-05 14:21:12 +0100
commitd1c9573ef4b54e03b66a9507a57104591078d35b (patch)
tree38dbcbe99a5b76d322d1634014bc390e75b01c20 /src
parentUnity build fixes (#253) (diff)
downloadzen-d1c9573ef4b54e03b66a9507a57104591078d35b.tar.xz
zen-d1c9573ef4b54e03b66a9507a57104591078d35b.zip
projectstore getchunks rpc with modtag (#244)
Feature: Project store "getchunks" rpc call /prj/{project}/oplog/{log}/rpc extended to accept both CAS (RawHash) and Id (Oid) identifiers as well as partial ranges
Diffstat (limited to 'src')
-rw-r--r--src/zencore/filesystem.cpp24
-rw-r--r--src/zencore/include/zencore/compactbinarypackage.h4
-rw-r--r--src/zencore/include/zencore/filesystem.h4
-rw-r--r--src/zencore/include/zencore/testutils.h1
-rw-r--r--src/zencore/testutils.cpp20
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp12
-rw-r--r--src/zenserver/projectstore/projectstore.cpp1931
-rw-r--r--src/zenserver/projectstore/projectstore.h34
-rw-r--r--src/zenserver/projectstore/zenremoteprojectstore.cpp16
-rw-r--r--src/zenserver/vfs/vfsimpl.cpp3
-rw-r--r--src/zenstore/compactcas.cpp32
11 files changed, 1848 insertions, 233 deletions
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 36147c5a9..52f2c4adc 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -1435,12 +1435,34 @@ FileSizeFromHandle(void* NativeHandle)
int Fd = int(intptr_t(NativeHandle));
struct stat Stat;
fstat(Fd, &Stat);
- FileSize = size_t(Stat.st_size);
+ FileSize = size_t(Stat.st_size);
#endif
return FileSize;
}
+uint64_t
+GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+ FILETIME LastWriteTime;
+ BOOL OK = GetFileTime((HANDLE)NativeHandle, NULL, NULL, &LastWriteTime);
+ if (OK)
+ {
+ return ((uint64_t(LastWriteTime.dwHighDateTime) << 32) | LastWriteTime.dwLowDateTime);
+ }
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+ int Fd = int(uintptr_t(NativeHandle));
+ struct stat Stat;
+ if (0 == fstat(Fd, &Stat))
+ {
+ return gsl::narrow<uint64_t>(Stat.st_mtime);
+ }
+#endif
+ Ec = MakeErrorCodeFromLastError();
+ return 0;
+}
+
std::filesystem::path
GetRunningExecutablePath()
{
diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h
index 064481f83..12fcc41b7 100644
--- a/src/zencore/include/zencore/compactbinarypackage.h
+++ b/src/zencore/include/zencore/compactbinarypackage.h
@@ -64,6 +64,9 @@ public:
ZENCORE_API CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash);
ZENCORE_API CbAttachment(CompressedBuffer&& InValue, const IoHash& Hash);
+ /** Construct a binary attachment. Value is cloned if not owned. */
+ ZENCORE_API CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash);
+
/** Reset this to a null attachment. */
inline void Reset() { *this = CbAttachment(); }
@@ -130,7 +133,6 @@ public:
private:
ZENCORE_API CbAttachment(const CbObject& Value, const IoHash* Hash);
ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue);
- ZENCORE_API CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash);
IoHash Hash;
std::variant<std::nullptr_t, CbObject, CompositeBuffer, CompressedBuffer> Value;
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index 2cd663afb..dba4981f0 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -45,6 +45,10 @@ ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, st
*/
ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle);
+/** Get a native time tick of last modification time
+ */
+ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec);
+
ZENCORE_API std::filesystem::path GetRunningExecutablePath();
/** Set the max open file handle count to max allowed for the current process on Linux and MacOS
diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h
index 6a1c0184b..45fde4eda 100644
--- a/src/zencore/include/zencore/testutils.h
+++ b/src/zencore/include/zencore/testutils.h
@@ -33,6 +33,7 @@ struct ScopedCurrentDirectoryChange
};
IoBuffer CreateRandomBlob(uint64_t Size);
+IoBuffer CreateSemiRandomBlob(uint64_t Size);
struct FalseType
{
diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp
index d4c8aeaef..641d5508a 100644
--- a/src/zencore/testutils.cpp
+++ b/src/zencore/testutils.cpp
@@ -71,6 +71,26 @@ CreateRandomBlob(uint64_t Size)
return Data;
};
+IoBuffer
+CreateSemiRandomBlob(uint64_t Size)
+{
+ IoBuffer Result(Size);
+ const size_t PartCount = (Size / (1u * 1024u * 64)) + 1;
+ const size_t PartSize = Size / PartCount;
+ auto Part = CreateRandomBlob(PartSize);
+ auto Remain = Result.GetMutableView().CopyFrom(Part.GetView());
+ while (Remain.GetSize() >= PartSize)
+ {
+ Remain = Remain.CopyFrom(Part.GetView());
+ }
+ if (Remain.GetSize() > 0)
+ {
+ auto RemainBuffer = CreateRandomBlob(Remain.GetSize());
+ Remain.CopyFrom(RemainBuffer.GetView());
+ }
+ return Result;
+};
+
} // namespace zen
#endif // ZEN_WITH_TESTS
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 22216e88d..09f4c7ee2 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -561,7 +561,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req)
for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex)
{
const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
- IoBuffer FoundChunk = FoundLog->FindChunk(RequestedChunk.ChunkId);
+ IoBuffer FoundChunk = FoundLog->FindChunk(RequestedChunk.ChunkId, nullptr);
if (FoundChunk)
{
if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1))
@@ -834,7 +834,7 @@ HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req)
CompositeBuffer Chunk;
HttpContentType ContentType;
std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk, ContentType);
+ m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk, ContentType, nullptr);
if (Result.first == HttpResponseCode::OK)
{
m_ProjectStats.ChunkHitCount++;
@@ -883,8 +883,8 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req)
case HttpVerb::kGet:
{
IoBuffer Value;
- std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value);
-
+ std::pair<HttpResponseCode, std::string> Result =
+ m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr);
if (Result.first == HttpResponseCode::OK)
{
m_ProjectStats.ChunkHitCount++;
@@ -991,7 +991,7 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
std::vector<IoHash> NeedList = FoundLog->CheckPendingChunkReferences(ChunkList, std::chrono::minutes(2));
- CbObjectWriter Cbo;
+ CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1);
Cbo.BeginArray("need");
{
for (const IoHash& Hash : NeedList)
@@ -1544,7 +1544,7 @@ LoadReferencedSet(ProjectStore::Oplog& Log)
return std::optional<OplogReferencedSet>();
}
- return OplogReferencedSet::LoadFromChunk(Log.FindChunk(ChunkId));
+ return OplogReferencedSet::LoadFromChunk(Log.FindChunk(ChunkId, nullptr));
}
void
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 415c80078..cd0b53394 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -314,6 +314,27 @@ namespace {
static_assert(sizeof(OplogIndexHeader) == 64);
+ static std::uint64_t GetModificationTagFromRawHash(const IoHash& Hash)
+ {
+ IoHash::Hasher H;
+ return H(Hash);
+ }
+
+ static std::uint64_t GetModificationTagFromModificationTime(IoBuffer FileBuffer)
+ {
+ IoBufferFileReference FileRef;
+ if (FileBuffer.GetFileReference(FileRef))
+ {
+ std::error_code Ec;
+ uint64_t ModificationTick = GetModificationTickFromHandle(FileRef.FileHandle, Ec);
+ if (!Ec)
+ {
+ return ModificationTick;
+ }
+ }
+ return {};
+ }
+
} // namespace
//////////////////////////////////////////////////////////////////////////
@@ -1920,19 +1941,27 @@ ProjectStore::Oplog::GetChunkByRawHash(const IoHash& RawHash)
}
bool
-ProjectStore::Oplog::IterateChunks(std::span<IoHash> RawHashes,
- const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
- WorkerThreadPool* OptionalWorkerPool,
- uint64_t LargeSizeLimit)
-{
- return m_CidStore.IterateChunks(RawHashes, AsyncCallback, OptionalWorkerPool, LargeSizeLimit);
+ProjectStore::Oplog::IterateChunks(std::span<IoHash> RawHashes,
+ bool IncludeModTag,
+ const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback,
+ WorkerThreadPool* OptionalWorkerPool,
+ uint64_t LargeSizeLimit)
+{
+ return m_CidStore.IterateChunks(
+ RawHashes,
+ [&](size_t Index, const IoBuffer& Payload) {
+ return AsyncCallback(Index, Payload, IncludeModTag ? GetModificationTagFromRawHash(RawHashes[Index]) : 0);
+ },
+ OptionalWorkerPool,
+ LargeSizeLimit);
}
bool
-ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
- const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
- WorkerThreadPool* OptionalWorkerPool,
- uint64_t LargeSizeLimit)
+ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
+ bool IncludeModTag,
+ const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback,
+ WorkerThreadPool* OptionalWorkerPool,
+ uint64_t LargeSizeLimit)
{
std::vector<size_t> CidChunkIndexes;
std::vector<IoHash> CidChunkHashes;
@@ -1960,12 +1989,6 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
}
}
}
- m_CidStore.IterateChunks(
- CidChunkHashes,
- [&](size_t Index, const IoBuffer& Payload) { return AsyncCallback(CidChunkIndexes[Index], Payload); },
- OptionalWorkerPool,
- LargeSizeLimit);
-
if (OptionalWorkerPool)
{
std::atomic_bool Result = true;
@@ -1979,7 +2002,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
}
WorkLatch.AddCount(1);
OptionalWorkerPool->ScheduleWork(
- [this, &WorkLatch, &ChunkIds, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() {
+ [this, &WorkLatch, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() {
auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
if (Result.load() == false)
{
@@ -1995,7 +2018,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[ChunkIndex], FilePath);
}
- if (!AsyncCallback(FileChunkIndex, Payload))
+ if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0))
{
Result.store(false);
}
@@ -2012,6 +2035,18 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
});
}
+ if (!CidChunkHashes.empty())
+ {
+ m_CidStore.IterateChunks(
+ CidChunkHashes,
+ [&](size_t Index, const IoBuffer& Payload) {
+ size_t CidChunkIndex = CidChunkIndexes[Index];
+ return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0);
+ },
+ OptionalWorkerPool,
+ LargeSizeLimit);
+ }
+
WorkLatch.CountDown();
WorkLatch.Wait();
@@ -2019,6 +2054,18 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
}
else
{
+ if (!CidChunkHashes.empty())
+ {
+ m_CidStore.IterateChunks(
+ CidChunkHashes,
+ [&](size_t Index, const IoBuffer& Payload) {
+ size_t CidChunkIndex = CidChunkIndexes[Index];
+ return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0);
+ },
+ OptionalWorkerPool,
+ LargeSizeLimit);
+ }
+
for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++)
{
size_t FileChunkIndex = FileChunkIndexes[ChunkIndex];
@@ -2026,7 +2073,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath);
if (Payload)
{
- bool Result = AsyncCallback(FileChunkIndex, Payload);
+ bool Result = AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0);
if (!Result)
{
return false;
@@ -2038,7 +2085,7 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
}
IoBuffer
-ProjectStore::Oplog::FindChunk(const Oid& ChunkId)
+ProjectStore::Oplog::FindChunk(const Oid& ChunkId, uint64_t* OptOutModificationTag)
{
RwLock::SharedLockScope OplogLock(m_OplogLock);
if (!m_Storage)
@@ -2051,7 +2098,12 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId)
IoHash ChunkHash = ChunkIt->second;
OplogLock.ReleaseNow();
- return m_CidStore.FindChunkByCid(ChunkHash);
+ IoBuffer Result = m_CidStore.FindChunkByCid(ChunkHash);
+ if (Result && OptOutModificationTag != nullptr)
+ {
+ *OptOutModificationTag = GetModificationTagFromRawHash(ChunkHash);
+ }
+ return Result;
}
if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end())
@@ -2065,6 +2117,10 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId)
{
ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkId, FilePath);
}
+ else if (OptOutModificationTag != nullptr)
+ {
+ *OptOutModificationTag = GetModificationTagFromModificationTime(Result);
+ }
return Result;
}
@@ -2073,7 +2129,12 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId)
IoHash ChunkHash = MetaIt->second;
OplogLock.ReleaseNow();
- return m_CidStore.FindChunkByCid(ChunkHash);
+ IoBuffer Result = m_CidStore.FindChunkByCid(ChunkHash);
+ if (Result && OptOutModificationTag != nullptr)
+ {
+ *OptOutModificationTag = GetModificationTagFromRawHash(ChunkHash);
+ }
+ return Result;
}
return {};
@@ -2109,7 +2170,7 @@ ProjectStore::Oplog::GetAllChunksInfo()
for (ChunkInfo& Info : InfoArray)
{
- if (IoBuffer Chunk = FindChunk(Info.ChunkId))
+ if (IoBuffer Chunk = FindChunk(Info.ChunkId, nullptr))
{
Info.ChunkSize = Chunk.GetSize();
}
@@ -3032,7 +3093,7 @@ ProjectStore::Project::WriteAccessTimes()
{
using namespace std::literals;
- CbObjectWriter Writer;
+ CbObjectWriter Writer(32 + (m_LastAccessTimes.size() * 16));
{
RwLock::SharedLockScope _(m_LastAccessTimesLock);
@@ -3972,7 +4033,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId,
FoundLog->IterateChunks(
Ids,
- [&](size_t Index, const IoBuffer& Payload) {
+ false,
+ [&](size_t Index, const IoBuffer& Payload, uint64_t /*ModTag*/) {
try
{
if (Payload)
@@ -4033,7 +4095,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId,
256u * 1024u);
}
- CbObjectWriter Response;
+ CbObjectWriter Response(64u + Count * ((WantsIdField ? (5 + sizeof(Oid::OidBits)) : 0) + (WantsServerPathField ? 64 : 0) +
+ (WantsClientPathField ? 64 : 0) + (WantsSizeField ? 16 : 0) + (WantsRawSizeField ? 16 : 0)));
Response.BeginArray("files"sv);
for (size_t Index = 0; Index < Count; Index++)
{
@@ -4102,7 +4165,16 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
std::vector<uint64_t> RawSizes;
std::vector<uint64_t> Sizes;
- size_t Count = 0;
+ size_t Count = 0;
+ size_t EstimatedCount = FoundLog->OplogCount();
+ if (WantsIdField)
+ {
+ Ids.reserve(EstimatedCount);
+ }
+ if (WantsRawHashField || WantsRawSizeField || WantsSizeField)
+ {
+ Hashes.reserve(EstimatedCount);
+ }
FoundLog->IterateChunkMap([&](const Oid& Id, const IoHash& Hash) {
if (WantsIdField)
{
@@ -4129,7 +4201,8 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();
(void)FoundLog->IterateChunks(
Hashes,
- [&](size_t Index, const IoBuffer& Payload) -> bool {
+ false,
+ [&](size_t Index, const IoBuffer& Payload, uint64_t /*ModTag*/) -> bool {
try
{
if (Payload)
@@ -4190,7 +4263,9 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
256u * 1024u);
}
- CbObjectWriter Response;
+ CbObjectWriter Response(64u + Count * ((WantsIdField ? (5 + sizeof(Oid::OidBits)) : 0) +
+ (WantsRawHashField ? (10 + sizeof(IoHash::Hash)) : 0) + (WantsSizeField ? 16 : 0) +
+ (WantsRawSizeField ? 16 : 0)));
Response.BeginArray("chunkinfos"sv);
for (size_t Index = 0; Index < Count; Index++)
@@ -4250,7 +4325,7 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId,
const Oid Obj = Oid::FromHexString(ChunkId);
- IoBuffer Chunk = FoundLog->FindChunk(Obj);
+ IoBuffer Chunk = FoundLog->FindChunk(Obj, nullptr);
if (!Chunk)
{
return {HttpResponseCode::NotFound, {}};
@@ -4284,7 +4359,8 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
uint64_t Size,
ZenContentType AcceptType,
CompositeBuffer& OutChunk,
- ZenContentType& OutContentType)
+ ZenContentType& OutContentType,
+ uint64_t* OptionalInOutModificationTag)
{
if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
{
@@ -4293,41 +4369,19 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
const Oid Obj = Oid::FromHexString(ChunkId);
- return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType);
+ return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType, OptionalInOutModificationTag);
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::GetChunkRange(const std::string_view ProjectId,
- const std::string_view OplogId,
- Oid ChunkId,
- uint64_t Offset,
- uint64_t Size,
- ZenContentType AcceptType,
- CompositeBuffer& OutChunk,
- ZenContentType& OutContentType)
+static std::pair<HttpResponseCode, std::string>
+ExtractRange(IoBuffer&& Chunk,
+ uint64_t Offset,
+ uint64_t Size,
+ ZenContentType AcceptType,
+ ZenContentType& OutContentType,
+ CompositeBuffer& OutChunk,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize)
{
- bool IsOffset = Offset != 0 || Size != ~(0ull);
-
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
-
- ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false);
- if (!FoundLog)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
- }
- Project->TouchOplog(OplogId);
-
- IoBuffer Chunk = FoundLog->FindChunk(ChunkId);
- if (!Chunk)
- {
- return {HttpResponseCode::NotFound, {}};
- }
-
OutContentType = Chunk.GetContentType();
if (OutContentType == ZenContentType::kCompressedBinary)
@@ -4335,26 +4389,47 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
IoHash RawHash;
uint64_t RawSize;
CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), RawHash, RawSize);
- ZEN_ASSERT(!Compressed.IsNull());
+ if (!Compressed)
+ {
+ return {HttpResponseCode::InternalServerError, "malformed compressed buffer"};
+ }
+
+ const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == RawSize));
- if (IsOffset)
+ if (IsFullRange)
+ {
+ if (AcceptType == ZenContentType::kBinary)
+ {
+ OutChunk = Compressed.DecompressToComposite();
+ OutContentType = ZenContentType::kBinary;
+ }
+ else
+ {
+ OutChunk = Compressed.GetCompressed();
+ }
+ OutRawSize = 0;
+ }
+ else
{
if (Size == ~(0ull) || (Offset + Size) > RawSize)
{
- Size = RawSize - Offset;
+ if (Offset < RawSize)
+ {
+ Size = RawSize - Offset;
+ }
+ else
+ {
+ Size = 0;
+ }
}
if (Size == 0)
{
- return {
- HttpResponseCode::NotFound,
- fmt::format("Chunk request for range outside of chunk '{}/{}'. Request: Chunk: {}, Offset: {}, Size: {}, ChunkSize: {}",
- ProjectId,
- OplogId,
- ChunkId,
- Offset,
- Size,
- RawSize)};
+ return {HttpResponseCode::NotFound,
+ fmt::format("Chunk request for range outside of compressed chunk. Offset: {}, Size: {}, ChunkSize: {}",
+ Offset,
+ Size,
+ RawSize)};
}
if (AcceptType == ZenContentType::kBinary)
@@ -4368,47 +4443,93 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
// The client will have to compensate for any offsets that do not land on an even block size multiple
OutChunk = Compressed.GetRange(Offset, Size).GetCompressed();
}
+ OutRawSize = RawSize;
+ }
+ OutRawHash = RawHash;
+ }
+ else
+ {
+ const uint64_t ChunkSize = Chunk.GetSize();
+
+ const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == ChunkSize));
+ if (IsFullRange)
+ {
+ OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk)));
+ OutRawSize = 0;
}
else
{
- if (AcceptType == ZenContentType::kBinary)
+ if (Size == ~(0ull) || (Offset + Size) > ChunkSize)
{
- OutChunk = Compressed.DecompressToComposite();
- OutContentType = ZenContentType::kBinary;
+ if (Offset < ChunkSize)
+ {
+ Size = ChunkSize - Offset;
+ }
+ else
+ {
+ Size = 0;
+ }
}
- else
+
+ if (Size == 0)
{
- OutChunk = Compressed.GetCompressed();
+ return {HttpResponseCode::NotFound,
+ fmt::format("Chunk request for range outside of chunk. Offset: {}, Size: {}, ChunkSize: {}", Offset, Size, Size)};
}
+
+ OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size)));
+ OutRawSize = ChunkSize;
}
}
- else if (IsOffset)
+ return {HttpResponseCode::OK, {}};
+}
+
+std::pair<HttpResponseCode, std::string>
+ProjectStore::GetChunkRange(const std::string_view ProjectId,
+ const std::string_view OplogId,
+ Oid ChunkId,
+ uint64_t Offset,
+ uint64_t Size,
+ ZenContentType AcceptType,
+ CompositeBuffer& OutChunk,
+ ZenContentType& OutContentType,
+ uint64_t* OptionalInOutModificationTag)
+{
+ Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
+ if (!Project)
{
- if (Size == ~(0ull) || (Offset + Size) > Chunk.GetSize())
- {
- Size = Chunk.GetSize() - Offset;
- }
+ return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)};
+ }
+ Project->TouchProject();
- if (Size == 0)
- {
- return {HttpResponseCode::NotFound,
- fmt::format("Chunk request for range outside of chunk '{}/{}'. Request: Chunk: {}, Offset: {}, Size: {}, ChunkSize: {}",
- ProjectId,
- OplogId,
- ChunkId,
- Offset,
- Size,
- Chunk.GetSize())};
- }
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false);
+ if (!FoundLog)
+ {
+ return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
+ }
+ Project->TouchOplog(OplogId);
- OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size)));
+ uint64_t OldTag = OptionalInOutModificationTag == nullptr ? 0 : *OptionalInOutModificationTag;
+ IoBuffer Chunk = FoundLog->FindChunk(ChunkId, OptionalInOutModificationTag);
+ if (!Chunk)
+ {
+ return {HttpResponseCode::NotFound, {}};
}
- else
+ if (OptionalInOutModificationTag != nullptr && OldTag == *OptionalInOutModificationTag)
{
- OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk)));
+ return {HttpResponseCode::NotModified, {}};
}
- return {HttpResponseCode::OK, {}};
+ IoHash _;
+ uint64_t __;
+ std::pair<HttpResponseCode, std::string> Result =
+ ExtractRange(std::move(Chunk), Offset, Size, AcceptType, OutContentType, OutChunk, /*OutRawHash*/ _, /*OutRawSize*/ __);
+ if (Result.first != HttpResponseCode::OK)
+ {
+ return {Result.first,
+ fmt::format("Chunk request for chunk {} in {}/{} failed. Reason: '{}'", ChunkId, ProjectId, OplogId, Result.second)};
+ }
+ return Result;
}
std::pair<HttpResponseCode, std::string>
@@ -4416,7 +4537,8 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view Cid,
ZenContentType AcceptType,
- IoBuffer& OutChunk)
+ IoBuffer& OutChunk,
+ uint64_t* OptionalInOutModificationTag)
{
Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
if (!Project)
@@ -4445,6 +4567,16 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
return {HttpResponseCode::NotFound, fmt::format("chunk - '{}' MISSING", Cid)};
}
+ if (OptionalInOutModificationTag != nullptr)
+ {
+ uint64_t OldTag = *OptionalInOutModificationTag;
+ *OptionalInOutModificationTag = GetModificationTagFromRawHash(Hash);
+ if (*OptionalInOutModificationTag == OldTag)
+ {
+ return {HttpResponseCode::NotModified, {}};
+ }
+ }
+
if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary)
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk));
@@ -4504,6 +4636,354 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
}
std::pair<HttpResponseCode, std::string>
+ProjectStore::GetChunks(const std::string_view ProjectId,
+ const std::string_view OplogId,
+ const CbObject& RequestObject,
+ CbPackage& OutResponsePackage)
+{
+ ZEN_TRACE_CPU("Store::GetChunks");
+
+ using namespace std::literals;
+
+ Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
+ if (!Project)
+ {
+ return {HttpResponseCode::NotFound, fmt::format("getchunks rpc request for unknown project '{}'", ProjectId)};
+ }
+ Project->TouchProject();
+
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
+ if (!FoundLog)
+ {
+ return {HttpResponseCode::NotFound, fmt::format("getchunks rpc request for unknown oplog '{}/{}'", ProjectId, OplogId)};
+ }
+ Project->TouchOplog(OplogId);
+
+ if (RequestObject["chunks"sv].IsArray())
+ {
+ // Legacy full chunks only by rawhash
+
+ CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
+
+ CbObjectWriter ResponseWriter;
+ ResponseWriter.BeginArray("chunks"sv);
+ for (CbFieldView FieldView : ChunksArray)
+ {
+ IoHash RawHash = FieldView.AsHash();
+ IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash);
+ if (ChunkBuffer)
+ {
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer));
+ if (Compressed)
+ {
+ ResponseWriter.AddHash(RawHash);
+ OutResponsePackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash));
+ }
+ else
+ {
+ ZEN_WARN("oplog '{}/{}': invalid compressed binary in cas store for {}", ProjectId, OplogId, RawHash);
+ }
+ }
+ }
+ ResponseWriter.EndArray();
+ OutResponsePackage.SetObject(ResponseWriter.Save());
+ return {HttpResponseCode::OK, {}};
+ }
+ else if (auto RequestFieldView = RequestObject["Request"sv]; RequestFieldView.IsObject())
+ {
+ CbObjectView RequestView = RequestFieldView.AsObjectView();
+ bool SkipData = RequestView["SkipData"].AsBool(false);
+ CbArrayView ChunksArray = RequestView["Chunks"sv].AsArrayView();
+
+ struct Request
+ {
+ struct InputData
+ {
+ uint64_t Offset = 0;
+ uint64_t Size = (uint64_t)-1;
+ std::variant<IoHash, Oid> Id;
+ std::optional<uint64_t> ModTag;
+ } Input;
+ struct OutputData
+ {
+ bool Exists = false;
+ IoBuffer ChunkBuffer;
+ uint64_t ModTag = 0;
+ } Output;
+ };
+
+ std::vector<Request> Requests;
+ size_t RequestCount = ChunksArray.Num();
+ if (RequestCount > 0)
+ {
+ Requests.reserve(RequestCount);
+ std::vector<IoHash> ChunkRawHashes;
+ std::vector<size_t> ChunkRawHashesRequestIndex;
+ std::vector<Oid> ChunkIds;
+ std::vector<size_t> ChunkIdsRequestIndex;
+ bool DoBatch = RequestCount > 1;
+ if (DoBatch)
+ {
+ ChunkRawHashes.reserve(RequestCount);
+ ChunkRawHashesRequestIndex.reserve(RequestCount);
+ ChunkIds.reserve(RequestCount);
+ ChunkIdsRequestIndex.reserve(RequestCount);
+ }
+ for (CbFieldView FieldView : ChunksArray)
+ {
+ CbObjectView ChunkObject = FieldView.AsObjectView();
+ Request ChunkRequest = {
+ .Input{.Offset = ChunkObject["Offset"sv].AsUInt64(0), .Size = ChunkObject["Size"sv].AsUInt64((uint64_t)-1)}};
+ if (CbFieldView InputModificationTagView = ChunkObject.FindView("ModTag"); InputModificationTagView.IsInteger())
+ {
+ ChunkRequest.Input.ModTag = InputModificationTagView.AsUInt64();
+ }
+ if (CbFieldView RawHashView = ChunkObject.FindView("RawHash"sv); RawHashView.IsHash())
+ {
+ const IoHash ChunkHash = RawHashView.AsHash();
+ ChunkRequest.Input.Id = ChunkHash;
+ if (DoBatch)
+ {
+ ChunkRawHashes.push_back(ChunkHash);
+ ChunkRawHashesRequestIndex.push_back(Requests.size());
+ }
+ }
+ else if (CbFieldView IdView = ChunkObject.FindView("Oid"sv); IdView.IsObjectId())
+ {
+ const Oid ChunkId = IdView.AsObjectId();
+ ChunkRequest.Input.Id = ChunkId;
+ if (DoBatch)
+ {
+ ChunkIds.push_back(ChunkId);
+ ChunkIdsRequestIndex.push_back(Requests.size());
+ }
+ }
+ else
+ {
+ return {HttpResponseCode::BadRequest,
+ fmt::format("oplog '{}/{}': malformed getchunks rpc request object, chunk request has no identifier",
+ ProjectId,
+ OplogId)};
+ }
+ Requests.emplace_back(std::move(ChunkRequest));
+ }
+
+ if (DoBatch)
+ {
+ WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();
+ if (!ChunkRawHashes.empty())
+ {
+ FoundLog->IterateChunks(
+ ChunkRawHashes,
+ true,
+ [&](size_t Index, const IoBuffer& Payload, uint64_t ModTag) -> bool {
+ if (Payload)
+ {
+ size_t RequestIndex = ChunkRawHashesRequestIndex[Index];
+ Requests[RequestIndex].Output.Exists = true;
+ if (!SkipData)
+ {
+ Requests[RequestIndex].Output.ChunkBuffer = Payload;
+ Requests[RequestIndex].Output.ChunkBuffer.MakeOwned();
+ }
+ Requests[RequestIndex].Output.ModTag = ModTag;
+ }
+ return true;
+ },
+ &WorkerPool,
+ 8u * 1024);
+ }
+ if (!ChunkIdsRequestIndex.empty())
+ {
+ FoundLog->IterateChunks(
+ ChunkIds,
+ true,
+ [&](size_t Index, const IoBuffer& Payload, uint64_t ModTag) -> bool {
+ if (Payload)
+ {
+ size_t RequestIndex = ChunkIdsRequestIndex[Index];
+ Requests[RequestIndex].Output.Exists = true;
+ if (!SkipData)
+ {
+ Requests[RequestIndex].Output.ChunkBuffer = Payload;
+ Requests[RequestIndex].Output.ChunkBuffer.MakeOwned();
+ }
+ Requests[RequestIndex].Output.ModTag = ModTag;
+ }
+ return true;
+ },
+ &WorkerPool,
+ 8u * 1024);
+ }
+ }
+ else
+ {
+ Request& ChunkRequest = Requests.front();
+ if (ChunkRequest.Input.Id.index() == 0)
+ {
+ const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id);
+ IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash);
+ if (Payload)
+ {
+ ChunkRequest.Output.Exists = true;
+ ChunkRequest.Output.ModTag = GetModificationTagFromRawHash(ChunkHash);
+ if (!SkipData)
+ {
+ ChunkRequest.Output.ChunkBuffer = Payload;
+ }
+ }
+ }
+ else
+ {
+ const Oid& ChunkId = std::get<Oid>(ChunkRequest.Input.Id);
+ uint64_t ModTag = 0;
+ IoBuffer Payload = FoundLog->FindChunk(ChunkId, &ModTag);
+ if (Payload)
+ {
+ ChunkRequest.Output.Exists = true;
+ ChunkRequest.Output.ModTag = ModTag;
+ if (!SkipData)
+ {
+ ChunkRequest.Output.ChunkBuffer = Payload;
+ }
+ }
+ }
+ }
+ }
+
+ CbObjectWriter ResponseWriter(32 + Requests.size() * 64u);
+ ResponseWriter.BeginArray("Chunks"sv);
+ {
+ for (Request& ChunkRequest : Requests)
+ {
+ if (ChunkRequest.Output.Exists)
+ {
+ ResponseWriter.BeginObject();
+ {
+ if (ChunkRequest.Input.Id.index() == 0)
+ {
+ const IoHash& RawHash = std::get<IoHash>(ChunkRequest.Input.Id);
+ ResponseWriter.AddHash("Id", RawHash);
+ }
+ else
+ {
+ const Oid& Id = std::get<Oid>(ChunkRequest.Input.Id);
+ ResponseWriter.AddObjectId("Id", Id);
+ }
+ if (!ChunkRequest.Input.ModTag.has_value() || ChunkRequest.Input.ModTag.value() != ChunkRequest.Output.ModTag)
+ {
+ ResponseWriter.AddInteger("ModTag", ChunkRequest.Output.ModTag);
+ if (!SkipData)
+ {
+ CompositeBuffer ChunkRange;
+ ZenContentType ContentType;
+ IoHash FullChunkRawHash;
+ uint64_t FullChunkSize = 0;
+ auto ExtractRangeResult = ExtractRange(std::move(ChunkRequest.Output.ChunkBuffer),
+ ChunkRequest.Input.Offset,
+ ChunkRequest.Input.Size,
+ ZenContentType::kCompressedBinary,
+ ContentType,
+ ChunkRange,
+ FullChunkRawHash,
+ FullChunkSize);
+ if (ExtractRangeResult.first == HttpResponseCode::OK)
+ {
+ if (ContentType == ZenContentType::kCompressedBinary)
+ {
+ ZEN_ASSERT(FullChunkRawHash != IoHash::Zero);
+ CompressedBuffer CompressedValue =
+ CompressedBuffer::FromCompressedNoValidate(std::move(ChunkRange));
+ ZEN_ASSERT(CompressedValue);
+
+ if (FullChunkSize != 0)
+ {
+ // This really could use some thought so we don't send the same data if we get a request for
+ // multiple ranges from the same chunk block
+
+ uint64_t FragmentRawOffset = 0;
+ OodleCompressor Compressor;
+ OodleCompressionLevel CompressionLevel;
+ uint64_t BlockSize = 0;
+ if (CompressedValue.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize))
+ {
+ if (BlockSize > 0)
+ {
+ FragmentRawOffset = (ChunkRequest.Input.Offset / BlockSize) * BlockSize;
+ }
+ else
+ {
+ FragmentRawOffset = ChunkRequest.Input.Offset;
+ }
+ uint64_t FragmentRawLength = CompressedValue.DecodeRawSize();
+
+ IoHashStream FragmentHashStream;
+ FragmentHashStream.Append(FullChunkRawHash.Hash, sizeof(FullChunkRawHash.Hash));
+ FragmentHashStream.Append(&FragmentRawOffset, sizeof(FragmentRawOffset));
+ FragmentHashStream.Append(&FragmentRawLength, sizeof(FragmentRawLength));
+ IoHash FragmentHash = FragmentHashStream.GetHash();
+
+ ResponseWriter.AddHash("FragmentHash", FragmentHash);
+ ResponseWriter.AddInteger("FragmentOffset", FragmentRawOffset);
+ ResponseWriter.AddInteger("RawSize", FullChunkSize);
+ OutResponsePackage.AddAttachment(CbAttachment(CompressedValue, FragmentHash));
+ }
+ else
+ {
+ std::string ErrorString =
+ "Failed to get compression parameters from partial compressed buffer";
+ ResponseWriter.AddString("Error", ErrorString);
+ ZEN_WARN("oplog '{}/{}': {}", ProjectId, OplogId, ErrorString);
+ }
+ }
+ else
+ {
+ ResponseWriter.AddHash("RawHash"sv, FullChunkRawHash);
+ OutResponsePackage.AddAttachment(CbAttachment(std::move(CompressedValue), FullChunkRawHash));
+ }
+ }
+ else
+ {
+ IoHashStream HashStream;
+ ZEN_ASSERT(ChunkRequest.Input.Id.index() == 1);
+ const Oid& Id = std::get<Oid>(ChunkRequest.Input.Id);
+ HashStream.Append(Id.OidBits, sizeof(Id.OidBits));
+ HashStream.Append(&ChunkRequest.Input.Offset, sizeof(ChunkRequest.Input.Offset));
+ HashStream.Append(&ChunkRequest.Input.Size, sizeof(ChunkRequest.Input.Size));
+ IoHash Hash = HashStream.GetHash();
+
+ ResponseWriter.AddHash("Hash"sv, Hash);
+ if (FullChunkSize != 0)
+ {
+ ResponseWriter.AddInteger("Size", FullChunkSize);
+ }
+ OutResponsePackage.AddAttachment(CbAttachment(std::move(ChunkRange), Hash));
+ }
+ }
+ else
+ {
+ std::string ErrorString = fmt::format("Failed fetchiong chunk range ({})", ExtractRangeResult.second);
+ ResponseWriter.AddString("Error", ErrorString);
+ ZEN_WARN("oplog '{}/{}': {}", ProjectId, OplogId, ErrorString);
+ }
+ }
+ }
+ }
+ ResponseWriter.EndObject();
+ }
+ }
+ }
+ ResponseWriter.EndArray();
+ OutResponsePackage.SetObject(ResponseWriter.Save());
+ return {HttpResponseCode::OK, {}};
+ }
+ else
+ {
+ return {HttpResponseCode::BadRequest, fmt::format("oplog '{}/{}': malformed getchunks rpc request object", ProjectId, OplogId)};
+ }
+}
+
+std::pair<HttpResponseCode, std::string>
ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_view OplogId, IoBuffer&& Payload, CbObject& OutResponse)
{
ZEN_TRACE_CPU("Store::WriteOplog");
@@ -4570,7 +5050,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
return ConvertResult(RemoteResult);
}
- CbObjectWriter Cbo;
+ CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1);
Cbo.BeginArray("need");
{
for (const IoHash& Hash : Attachments)
@@ -4767,34 +5247,34 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
else if (Method == "getchunks"sv)
{
ZEN_TRACE_CPU("Store::Rpc::getchunks");
- CbPackage ResponsePackage;
+
+ RpcAcceptOptions AcceptFlags = static_cast<RpcAcceptOptions>(Cb["AcceptFlags"sv].AsUInt16(0u));
+ // TODO: Move OpenProccessCache to a more general place
+ // int32_t TargetProcessId = Cb["Pid"sv].AsInt32(0);
+
+ CbPackage ResponsePackage;
+ std::pair<HttpResponseCode, std::string> Result = GetChunks(ProjectId, OplogId, Cb, ResponsePackage);
+ if (Result.first == HttpResponseCode::OK)
{
- CbArrayView ChunksArray = Cb["chunks"sv].AsArrayView();
- CbObjectWriter ResponseWriter;
- ResponseWriter.BeginArray("chunks"sv);
- for (CbFieldView FieldView : ChunksArray)
+ void* TargetProcessHandle = nullptr;
+ FormatFlags Flags = FormatFlags::kDefault;
+ if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences))
{
- IoHash RawHash = FieldView.AsHash();
- IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash);
- if (ChunkBuffer)
+ Flags |= FormatFlags::kAllowLocalReferences;
+ if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences))
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer));
- if (Compressed)
- {
- ResponseWriter.AddHash(RawHash);
- ResponsePackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash));
- }
- else
- {
- ZEN_WARN("oplog '{}/{}': invalid compressed binary in cas store for {}", ProjectId, OplogId, RawHash);
- }
+ Flags |= FormatFlags::kDenyPartialLocalReferences;
}
+ // TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(Context.SessionId, TargetPid);
}
- ResponseWriter.EndArray();
- ResponsePackage.SetObject(ResponseWriter.Save());
+
+ CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage, Flags, TargetProcessHandle);
+ HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
+ }
+ else
+ {
+ HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
}
- CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage, FormatFlags::kDefault);
- HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
return true;
}
else if (Method == "putchunks"sv)
@@ -4927,7 +5407,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
++TotalFiles;
// Rewrite file array entry with new data reference
- CbObjectWriter Writer;
+ CbObjectWriter Writer(View.GetSize());
RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool {
if (Field.GetName() == "data"sv)
{
@@ -6012,7 +6492,7 @@ namespace testutils {
return OidStringBuilder.ToString();
}
- CbPackage CreateOplogPackage(const Oid& Id, const std::span<const std::pair<Oid, CompressedBuffer>>& Attachments)
+ CbPackage CreateBulkDataOplogPackage(const Oid& Id, const std::span<const std::pair<Oid, CompressedBuffer>>& Attachments)
{
CbPackage Package;
CbObjectWriter Object;
@@ -6038,16 +6518,43 @@ namespace testutils {
return Package;
};
+ CbPackage CreateFilesOplogPackage(const Oid& Id,
+ const std::filesystem::path ProjectRootDir,
+ const std::span<const std::pair<Oid, std::filesystem::path>>& Attachments)
+ {
+ CbPackage Package;
+ CbObjectWriter Object;
+ Object << "key"sv << OidAsString(Id);
+ if (!Attachments.empty())
+ {
+ Object.BeginArray("files");
+ for (const auto& Attachment : Attachments)
+ {
+ std::filesystem::path ServerPath = std::filesystem::relative(Attachment.second, ProjectRootDir);
+ std::filesystem::path ClientPath = ServerPath; // dummy
+ Object.BeginObject();
+ Object << "id"sv << Attachment.first;
+ Object << "serverpath"sv << ServerPath.string();
+ Object << "clientpath"sv << ClientPath.string();
+ Object.EndObject();
+ }
+ Object.EndArray();
+ }
+ Package.SetObject(Object.Save());
+ return Package;
+ };
+
std::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments(
const std::span<const size_t>& Sizes,
- OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast)
+ OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
+ uint64_t BlockSize = 0)
{
std::vector<std::pair<Oid, CompressedBuffer>> Result;
Result.reserve(Sizes.size());
for (size_t Size : Sizes)
{
CompressedBuffer Compressed =
- CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(Size)), OodleCompressor::Mermaid, CompressionLevel);
+ CompressedBuffer::Compress(SharedBuffer(CreateSemiRandomBlob(Size)), OodleCompressor::Mermaid, CompressionLevel, BlockSize);
Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed));
}
return Result;
@@ -6069,6 +6576,63 @@ namespace testutils {
return 0;
}
+ template<typename ChunkType>
+ CbObject BuildChunksRequest(bool SkipData,
+ std::string_view IdName,
+ const std::vector<ChunkType>& Chunks,
+ const std::vector<std::pair<size_t, size_t>>& Ranges,
+ const std::vector<uint64_t>& ModTags)
+ {
+ CbObjectWriter Request;
+ Request.BeginObject("Request"sv);
+ {
+ if (SkipData)
+ {
+ Request.AddBool("SkipData"sv, true);
+ }
+ if (!Chunks.empty())
+ {
+ Request.BeginArray("Chunks");
+ for (size_t Index = 0; Index < Chunks.size(); Index++)
+ {
+ Request.BeginObject();
+ {
+ Request << IdName << Chunks[Index];
+ if (!ModTags.empty())
+ {
+ Request << "ModTag" << ModTags[Index];
+ }
+ if (!Ranges.empty())
+ {
+ Request << "Offset" << Ranges[Index].first;
+ Request << "Size" << Ranges[Index].second;
+ }
+ }
+ Request.EndObject();
+ }
+ Request.EndArray();
+ }
+ }
+ Request.EndObject();
+ return Request.Save();
+ };
+
+ CbObject BuildChunksRequest(bool SkipData,
+ const std::vector<Oid>& Chunks,
+ const std::vector<std::pair<size_t, size_t>>& Ranges,
+ const std::vector<uint64_t>& ModTags)
+ {
+ return BuildChunksRequest<Oid>(SkipData, "Oid", Chunks, Ranges, ModTags);
+ }
+
+ CbObject BuildChunksRequest(bool SkipData,
+ const std::vector<IoHash>& Chunks,
+ const std::vector<std::pair<size_t, size_t>>& Ranges,
+ const std::vector<uint64_t>& ModTags)
+ {
+ return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags);
+ }
+
} // namespace testutils
TEST_CASE("project.store.create")
@@ -6191,13 +6755,14 @@ TEST_CASE_TEMPLATE("project.store.export",
ProjectStore::Oplog* Oplog = Project->NewOplog("oplog1", {});
CHECK(Oplog != nullptr);
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77})));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99})));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122})));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {}));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77})));
Oplog->AppendNewOplogEntry(
- CreateOplogPackage(Oid::NewOid(),
- CreateAttachments(std::initializer_list<size_t>{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None)));
+ CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99})));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122})));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(
+ Oid::NewOid(),
+ CreateAttachments(std::initializer_list<size_t>{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None)));
FileRemoteStoreOptions Options = {
RemoteStoreOptions{.MaxBlockSize = 64u * 1024, .MaxChunkEmbedSize = 32 * 1024u, .ChunkFileSizeLimit = 64u * 1024u},
@@ -6325,10 +6890,11 @@ TEST_CASE("project.store.gc")
ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1", Project1OplogPath);
CHECK(Oplog != nullptr);
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77})));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99})));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122})));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {}));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77})));
+ Oplog->AppendNewOplogEntry(
+ CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99})));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122})));
}
{
@@ -6342,21 +6908,23 @@ TEST_CASE("project.store.gc")
ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog2", Project2Oplog1Path);
CHECK(Oplog != nullptr);
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177})));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {}));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177})));
+ Oplog->AppendNewOplogEntry(
+ CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96})));
Oplog->AppendNewOplogEntry(
- CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96})));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221})));
+ CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221})));
}
{
ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog3", Project2Oplog2Path);
CHECK(Oplog != nullptr);
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{137})));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {}));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{137})));
Oplog->AppendNewOplogEntry(
- CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9723, 683, 594, 98})));
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{531, 271})));
+ CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9723, 683, 594, 98})));
+ Oplog->AppendNewOplogEntry(
+ CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{531, 271})));
}
}
@@ -6507,7 +7075,7 @@ TEST_CASE("project.store.gc.prep")
Project1RootDir.string(),
Project1FilePath.string()));
ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath);
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments));
}
{
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
@@ -6538,7 +7106,7 @@ TEST_CASE("project.store.gc.prep")
// Make sure the chunks are stored but not the referencing op
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath);
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments));
Project1->DeleteOplog("oplog1"sv);
}
{
@@ -6585,7 +7153,7 @@ TEST_CASE("project.store.gc.prep")
{
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
ProjectStore::Oplog* Oplog = Project1->OpenOplog("oplog1"sv, true, true);
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments));
Oplog->RemovePendingChunkReferences(OpChunkHashes);
CHECK(Oplog->GetPendingChunkReferencesLocked().size() == 0);
}
@@ -6619,7 +7187,7 @@ TEST_CASE("project.store.gc.prep")
// Make sure the chunks are stored but not the referencing op
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath);
- Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), OpAttachments));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), OpAttachments));
Project1->DeleteOplog("oplog1"sv);
}
@@ -6674,6 +7242,927 @@ TEST_CASE("project.store.gc.prep")
}
}
+TEST_CASE("project.store.rpc.getchunks")
+{
+ using namespace std::literals;
+ using namespace testutils;
+
+ ScopedTemporaryDirectory TempDir;
+
+ auto JobQueue = MakeJobQueue(1, ""sv);
+ GcManager Gc;
+ CidStore CidStore(Gc);
+ CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
+ CidStore.Initialize(CidConfig);
+
+ std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{});
+ std::filesystem::path RootDir = TempDir.Path() / "root"sv;
+ std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
+
+ std::filesystem::path Project1RootDir = TempDir.Path() / "game1"sv;
+ std::filesystem::path Project1FilePath = TempDir.Path() / "game1"sv / "game.uproject"sv;
+ {
+ CreateDirectories(Project1FilePath.parent_path());
+ BasicFile ProjectFile;
+ ProjectFile.Open(Project1FilePath, BasicFile::Mode::kTruncate);
+ }
+
+ std::vector<Oid> OpIds;
+ OpIds.insert(OpIds.end(), {Oid::NewOid(), Oid::NewOid(), Oid::NewOid(), Oid::NewOid()});
+ std::unordered_map<Oid, std::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments;
+ Oid FilesOpId = Oid::NewOid();
+ std::vector<std::pair<Oid, std::filesystem::path>> FilesOpIdAttachments;
+ {
+ Ref<ProjectStore::Project> Project1(ProjectStore.NewProject(BasePath / "proj1"sv,
+ "proj1"sv,
+ RootDir.string(),
+ EngineRootDir.string(),
+ Project1RootDir.string(),
+ Project1FilePath.string()));
+ ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, {});
+ CHECK(Oplog != nullptr);
+ Attachments[OpIds[0]] = {};
+ Attachments[OpIds[1]] = CreateAttachments(std::initializer_list<size_t>{77});
+ Attachments[OpIds[2]] =
+ CreateAttachments(std::initializer_list<size_t>{200 * 1024, 314 * 1024, 690, 99}, OodleCompressionLevel::VeryFast, 128 * 1024);
+ Attachments[OpIds[3]] = CreateAttachments(std::initializer_list<size_t>{55, 122});
+ for (auto It : Attachments)
+ {
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(It.first, It.second));
+ }
+
+ std::filesystem::path UncompressedFilePath = RootDir / "content" / "uncompressed_file";
+ CreateDirectories(UncompressedFilePath.parent_path());
+ IoBuffer FileBlob = CreateRandomBlob(81823 * 2);
+ WriteFile(UncompressedFilePath, FileBlob);
+ FilesOpIdAttachments.push_back({Oid::NewOid(), UncompressedFilePath});
+ Oplog->AppendNewOplogEntry(CreateFilesOplogPackage(FilesOpId, RootDir, FilesOpIdAttachments));
+ }
+
+ // Invalid request
+ {
+ CbObjectWriter Request;
+ Request.BeginObject("WrongName"sv);
+ Request.EndObject();
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv, "oplog1"sv, Request.Save(), Response);
+ CHECK_EQ(HttpResponseCode::BadRequest, Result.first);
+ }
+
+ // Empty request
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, std::vector<IoHash>{}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(0, Chunks.Num());
+ }
+ // Single non-existing chunk by RawHash
+ IoHash NotFoundIoHash = IoHash::Max;
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {NotFoundIoHash}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(0, Chunks.Num());
+ }
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {NotFoundIoHash}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(0, Chunks.Num());
+ }
+ // Single non-existing chunk by Id
+ Oid NotFoundId = Oid::NewOid();
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {NotFoundId}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(0, Chunks.Num());
+ }
+ {
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv, "oplog1"sv, testutils::BuildChunksRequest(/*SkipData*/ true, {NotFoundId}, {}, {}), Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(0, Chunks.Num());
+ }
+ // Single existing chunk by RawHash
+ {
+ // Fresh fetch
+ IoHash FirstAttachmentHash = Attachments[OpIds[2]][1].second.DecodeRawHash();
+ uint64_t ResponseModTag = 0;
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {}),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(1, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ IoHash Id = Chunk["Id"].AsHash();
+ CHECK_EQ(FirstAttachmentHash, Id);
+ ResponseModTag = Chunk["ModTag"].AsUInt64();
+ CHECK_NE(0, ResponseModTag);
+ IoHash AttachmentHash = Chunk["RawHash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompressedBuffer Buffer = Attachment->AsCompressedBinary();
+ CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash());
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Fetch with matching ModTag
+ {
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {ResponseModTag}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ IoHash Id = Chunk["Id"].AsHash();
+ CHECK_EQ(FirstAttachmentHash, Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Fetch with mismatching ModTag
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks(
+ "proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentHash}, {}, {uint64_t(ResponseModTag + 1)}),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(1, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ IoHash Id = Chunk["Id"].AsHash();
+ CHECK_EQ(FirstAttachmentHash, Id);
+ ResponseModTag = Chunk["ModTag"].AsUInt64();
+ CHECK_NE(0, ResponseModTag);
+ IoHash AttachmentHash = Chunk["RawHash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompressedBuffer Buffer = Attachment->AsCompressedBinary();
+ CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash());
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Fresh modtime query
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ IoHash Id = Chunk["Id"].AsHash();
+ CHECK_EQ(FirstAttachmentHash, Id);
+ uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64();
+ CHECK_EQ(ResponseModTag, ResponseModTag2);
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Modtime query with matching ModTag
+ {
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {ResponseModTag}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ IoHash Id = Chunk["Id"].AsHash();
+ CHECK_EQ(FirstAttachmentHash, Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Modtime query with mismatching ModTag
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks(
+ "proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentHash}, {}, {uint64_t(ResponseModTag + 1)}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ IoHash Id = Chunk["Id"].AsHash();
+ CHECK_EQ(FirstAttachmentHash, Id);
+ uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64();
+ CHECK_EQ(ResponseModTag, ResponseModTag2);
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ // Single existing CID chunk by Id
+ {
+ Oid FirstAttachmentId = Attachments[OpIds[2]][1].first;
+ uint64_t ResponseModTag = 0;
+ {
+ // Full chunk request
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(1, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ ResponseModTag = Chunk["ModTag"].AsUInt64();
+ CHECK_NE(0, ResponseModTag);
+ IoHash AttachmentHash = Chunk["RawHash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompressedBuffer Buffer = Attachment->AsCompressedBinary();
+ CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash());
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ {
+ // Partial chunk request
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {{130 * 1024, 8100}}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(1, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ ResponseModTag = Chunk["ModTag"].AsUInt64();
+ CHECK_NE(0, ResponseModTag);
+ IoHash AttachmentHash = Chunk["FragmentHash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ std::uint64_t FragmentStart = Chunk["FragmentOffset"].AsUInt64();
+ CompressedBuffer Buffer = Attachment->AsCompressedBinary();
+ CHECK(FragmentStart <= 130 * 1024);
+ CHECK(FragmentStart + Buffer.DecodeRawSize() >= 130 * 1024 + 8100);
+ auto ResponseDecompressedBuffer = Buffer.Decompress(130 * 1024 - FragmentStart, 8100);
+ auto ExpectedDecompressedBuffer = Attachments[OpIds[2]][1].second.Decompress(130 * 1024, 8100);
+ CHECK(ResponseDecompressedBuffer.AsIoBuffer().GetView().EqualBytes(ExpectedDecompressedBuffer.AsIoBuffer().GetView()));
+ CHECK_EQ(Chunk["RawSize"sv].AsUInt64(), Attachments[OpIds[2]][1].second.DecodeRawSize());
+ CHECK(!Chunk.FindView("Size"));
+ }
+ {
+ // Fetch with matching ModTag
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {ResponseModTag}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ {
+ // Fetch with mismatching ModTag
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks(
+ "proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(1, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ uint64_t ResponseModTag3 = Chunk["ModTag"].AsUInt64();
+ CHECK_EQ(ResponseModTag, ResponseModTag3);
+ IoHash AttachmentHash = Chunk["RawHash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompressedBuffer Buffer = Attachment->AsCompressedBinary();
+ CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash());
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Fresh modtime query
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64();
+ CHECK_EQ(ResponseModTag, ResponseModTag2);
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Modtime query with matching ModTag
+ {
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {ResponseModTag}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Modtime query with mismatching ModTag
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks(
+ "proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64();
+ CHECK_EQ(ResponseModTag, ResponseModTag2);
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+
+ // Single existing file chunk by Id
+ {
+ Oid FirstAttachmentId = FilesOpIdAttachments[0].first;
+ uint64_t ResponseModTag = 0;
+ {
+ // Full chunk request
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(1, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ ResponseModTag = Chunk["ModTag"].AsUInt64();
+ CHECK_NE(0, ResponseModTag);
+ IoHash AttachmentHash = Chunk["Hash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompositeBuffer Buffer = Attachment->AsCompositeBinary();
+ CHECK_EQ(IoHash::HashBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten()), IoHash::HashBuffer(Buffer));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ {
+ // Partial chunk request
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {{81823, 5434}}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(1, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ ResponseModTag = Chunk["ModTag"].AsUInt64();
+ CHECK_NE(0, ResponseModTag);
+
+ IoHash AttachmentHash = Chunk["Hash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompositeBuffer Buffer = Attachment->AsCompositeBinary();
+ CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)),
+ IoHash::HashBuffer(Buffer));
+ CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ {
+ // Fetch with matching ModTag
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {ResponseModTag}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("Hash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ {
+ // Fetch with mismatching ModTag
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks(
+ "proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(1, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ uint64_t ResponseModTag3 = Chunk["ModTag"].AsUInt64();
+ CHECK_EQ(ResponseModTag, ResponseModTag3);
+ IoHash AttachmentHash = Chunk["Hash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompositeBuffer Buffer = Attachment->AsCompositeBinary();
+ CHECK_EQ(IoHash::HashBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten()), IoHash::HashBuffer(Buffer));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Fresh modtime query
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64();
+ CHECK_EQ(ResponseModTag, ResponseModTag2);
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Modtime query with matching ModTag
+ {
+ CbPackage Response;
+ auto Result =
+ ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {ResponseModTag}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("Hash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ // Modtime query with mismatching ModTag
+ {
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks(
+ "proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, {FirstAttachmentId}, {}, {uint64_t(ResponseModTag + 1)}),
+ Response);
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(1, Chunks.Num());
+ CbObjectView Chunk = (*begin(Chunks)).AsObjectView();
+
+ Oid Id = Chunk["Id"].AsObjectId();
+ CHECK_EQ(FirstAttachmentId, Id);
+ uint64_t ResponseModTag2 = Chunk["ModTag"].AsUInt64();
+ CHECK_EQ(ResponseModTag, ResponseModTag2);
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+
+ // Multi RawHash Request
+ {
+ std::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second,
+ Attachments[OpIds[2]][0].second,
+ Attachments[OpIds[2]][1].second};
+ std::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(),
+ AttachmentBuffers[1].DecodeRawHash(),
+ AttachmentBuffers[2].DecodeRawHash()};
+ std::vector<uint64_t> ResponseModTags(3, 0);
+ {
+ // Fresh fetch
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, AttachmentHashes, {}, {}),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(3, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ IoHash Id = Chunk["Id"].AsHash();
+
+ auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id);
+ CHECK(It != AttachmentHashes.end());
+ ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It);
+ CHECK_EQ(AttachmentHashes[Index], Id);
+ ResponseModTags[Index] = Chunk["ModTag"].AsUInt64();
+ CHECK_NE(0, ResponseModTags[Index]);
+ IoHash AttachmentHash = Chunk["RawHash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompressedBuffer Buffer = Attachment->AsCompressedBinary();
+ CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash());
+ CHECK(AttachmentBuffers[Index].GetCompressed().Flatten().GetView().EqualBytes(Buffer.GetCompressed().Flatten().GetView()));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ {
+ // Fetch with matching ModTag
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, AttachmentHashes, {}, ResponseModTags),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ IoHash Id = Chunk["Id"].AsHash();
+
+ auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id);
+ CHECK(It != AttachmentHashes.end());
+ ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It);
+ CHECK_EQ(AttachmentHashes[Index], Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ {
+ // Fresh modtime query
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, {}),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ IoHash Id = Chunk["Id"].AsHash();
+
+ auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id);
+ CHECK(It != AttachmentHashes.end());
+ ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It);
+ CHECK_EQ(AttachmentHashes[Index], Id);
+ CHECK_EQ(ResponseModTags[Index], Chunk["ModTag"].AsUInt64());
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ {
+ // Modtime query with matching ModTags
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, ResponseModTags),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ IoHash Id = Chunk["Id"].AsHash();
+
+ auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id);
+ CHECK(It != AttachmentHashes.end());
+ ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It);
+ CHECK_EQ(AttachmentHashes[Index], Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ {
+ // Modtime query with mismatching ModTags
+ std::vector<uint64_t> MismatchingModTags(ResponseModTags);
+ for (uint64_t& Tag : MismatchingModTags)
+ {
+ Tag++;
+ }
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, AttachmentHashes, {}, MismatchingModTags),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ IoHash Id = Chunk["Id"].AsHash();
+
+ auto It = std::find(AttachmentHashes.begin(), AttachmentHashes.end(), Id);
+ CHECK(It != AttachmentHashes.end());
+ ptrdiff_t Index = std::distance(AttachmentHashes.begin(), It);
+ CHECK_EQ(AttachmentHashes[Index], Id);
+ CHECK(Chunk["ModTag"].AsUInt64() == ResponseModTags[Index]);
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ }
+ // Multi Id Request
+ {
+ std::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second,
+ Attachments[OpIds[2]][0].second,
+ Attachments[OpIds[2]][1].second};
+ std::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(),
+ AttachmentBuffers[1].DecodeRawHash(),
+ AttachmentBuffers[2].DecodeRawHash()};
+ std::vector<Oid> AttachedIds{Attachments[OpIds[1]][0].first, Attachments[OpIds[2]][0].first, Attachments[OpIds[2]][1].first};
+ std::vector<uint64_t> ResponseModTags(3, 0);
+ {
+ // Fresh fetch
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, AttachedIds, {}, {}),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(3, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ Oid Id = Chunk["Id"].AsObjectId();
+
+ auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id);
+ CHECK(It != AttachedIds.end());
+ ptrdiff_t Index = std::distance(AttachedIds.begin(), It);
+ CHECK_EQ(AttachedIds[Index], Id);
+ ResponseModTags[Index] = Chunk["ModTag"].AsUInt64();
+ CHECK_NE(0, ResponseModTags[Index]);
+ IoHash AttachmentHash = Chunk["RawHash"].AsHash();
+ const CbAttachment* Attachment = Response.FindAttachment(AttachmentHash);
+ CHECK_NE(nullptr, Attachment);
+ CompressedBuffer Buffer = Attachment->AsCompressedBinary();
+ CHECK_EQ(AttachmentHash, Buffer.DecodeRawHash());
+ CHECK(AttachmentBuffers[Index].GetCompressed().Flatten().GetView().EqualBytes(Buffer.GetCompressed().Flatten().GetView()));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ {
+ // Fetch with matching ModTag
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ false, AttachedIds, {}, ResponseModTags),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ Oid Id = Chunk["Id"].AsObjectId();
+
+ auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id);
+ CHECK(It != AttachedIds.end());
+ ptrdiff_t Index = std::distance(AttachedIds.begin(), It);
+ CHECK_EQ(AttachedIds[Index], Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ {
+ // Fresh modtime query
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, {}),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ Oid Id = Chunk["Id"].AsObjectId();
+
+ auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id);
+ CHECK(It != AttachedIds.end());
+ ptrdiff_t Index = std::distance(AttachedIds.begin(), It);
+ CHECK_EQ(AttachedIds[Index], Id);
+ CHECK_EQ(ResponseModTags[Index], Chunk["ModTag"].AsUInt64());
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ {
+ // Modtime query with matching ModTags
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, ResponseModTags),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ Oid Id = Chunk["Id"].AsObjectId();
+
+ auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id);
+ CHECK(It != AttachedIds.end());
+ ptrdiff_t Index = std::distance(AttachedIds.begin(), It);
+ CHECK_EQ(AttachedIds[Index], Id);
+ CHECK(!Chunk.FindView("ModTag"));
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ {
+ // Modtime query with mismatching ModTags
+ std::vector<uint64_t> MismatchingModTags(ResponseModTags);
+ for (uint64_t& Tag : MismatchingModTags)
+ {
+ Tag++;
+ }
+ CbPackage Response;
+ auto Result = ProjectStore.GetChunks("proj1"sv,
+ "oplog1"sv,
+ testutils::BuildChunksRequest(/*SkipData*/ true, AttachedIds, {}, MismatchingModTags),
+ Response);
+
+ CHECK_EQ(HttpResponseCode::OK, Result.first);
+ CHECK_EQ(0, Response.GetAttachments().size());
+ CbArrayView Chunks = Response.GetObject()["Chunks"].AsArrayView();
+ CHECK_EQ(3, Chunks.Num());
+ for (CbFieldView ChunkView : Chunks)
+ {
+ CbObjectView Chunk = ChunkView.AsObjectView();
+ Oid Id = Chunk["Id"].AsObjectId();
+
+ auto It = std::find(AttachedIds.begin(), AttachedIds.end(), Id);
+ CHECK(It != AttachedIds.end());
+ ptrdiff_t Index = std::distance(AttachedIds.begin(), It);
+ CHECK_EQ(AttachedIds[Index], Id);
+ CHECK(Chunk["ModTag"].AsUInt64() == ResponseModTags[Index]);
+ CHECK(!Chunk.FindView("RawHash"));
+ CHECK(!Chunk.FindView("Size"));
+ CHECK(!Chunk.FindView("RawSize"));
+ }
+ }
+ }
+}
+
TEST_CASE("project.store.partial.read")
{
using namespace std::literals;
@@ -6718,63 +8207,119 @@ TEST_CASE("project.store.partial.read")
Attachments[OpIds[3]] = CreateAttachments(std::initializer_list<size_t>{55, 122});
for (auto It : Attachments)
{
- Oplog->AppendNewOplogEntry(CreateOplogPackage(It.first, It.second));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(It.first, It.second));
}
}
{
+ uint64_t ModificationTag = 0;
IoBuffer Chunk;
CHECK(ProjectStore
.GetChunk("proj1"sv,
"oplog1"sv,
Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(),
HttpContentType::kCompressedBinary,
- Chunk)
+ Chunk,
+ &ModificationTag)
.first == HttpResponseCode::OK);
IoHash RawHash;
uint64_t RawSize;
CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize());
- }
- CompositeBuffer ChunkResult;
- HttpContentType ContentType;
- CHECK(ProjectStore
- .GetChunkRange("proj1"sv,
- "oplog1"sv,
- OidAsString(Attachments[OpIds[2]][1].first),
- 0,
- ~0ull,
- HttpContentType::kCompressedBinary,
- ChunkResult,
- ContentType)
- .first == HttpResponseCode::OK);
- CHECK(ChunkResult);
- CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() ==
- Attachments[OpIds[2]][1].second.DecodeRawSize());
-
- CompositeBuffer PartialChunkResult;
- CHECK(ProjectStore
- .GetChunkRange("proj1"sv,
- "oplog1"sv,
- OidAsString(Attachments[OpIds[2]][1].first),
- 5,
- 1773,
- HttpContentType::kCompressedBinary,
- PartialChunkResult,
- ContentType)
- .first == HttpResponseCode::OK);
- CHECK(PartialChunkResult);
- IoHash PartialRawHash;
- uint64_t PartialRawSize;
- CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize);
- CHECK(PartialRawSize >= 1773);
-
- uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5);
- SharedBuffer PartialDecompressed = PartialCompressedResult.Decompress(RawOffsetInPartialCompressed);
- SharedBuffer FullDecompressed = Attachments[OpIds[2]][1].second.Decompress();
- const uint8_t* FullDataPtr = &(reinterpret_cast<const uint8_t*>(FullDecompressed.GetView().GetData())[5]);
- const uint8_t* PartialDataPtr = reinterpret_cast<const uint8_t*>(PartialDecompressed.GetView().GetData());
- CHECK(FullDataPtr[0] == PartialDataPtr[0]);
+ CHECK(ModificationTag != 0);
+ CHECK(ProjectStore
+ .GetChunk("proj1"sv,
+ "oplog1"sv,
+ Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(),
+ HttpContentType::kCompressedBinary,
+ Chunk,
+ &ModificationTag)
+ .first == HttpResponseCode::NotModified);
+ }
+
+ {
+ uint64_t FullChunkModificationTag = 0;
+ {
+ CompositeBuffer ChunkResult;
+ HttpContentType ContentType;
+ CHECK(ProjectStore
+ .GetChunkRange("proj1"sv,
+ "oplog1"sv,
+ OidAsString(Attachments[OpIds[2]][1].first),
+ 0,
+ ~0ull,
+ HttpContentType::kCompressedBinary,
+ ChunkResult,
+ ContentType,
+ &FullChunkModificationTag)
+ .first == HttpResponseCode::OK);
+ CHECK(ChunkResult);
+ CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() ==
+ Attachments[OpIds[2]][1].second.DecodeRawSize());
+ }
+ {
+ CompositeBuffer ChunkResult;
+ HttpContentType ContentType;
+ CHECK(ProjectStore
+ .GetChunkRange("proj1"sv,
+ "oplog1"sv,
+ OidAsString(Attachments[OpIds[2]][1].first),
+ 0,
+ ~0ull,
+ HttpContentType::kCompressedBinary,
+ ChunkResult,
+ ContentType,
+ &FullChunkModificationTag)
+ .first == HttpResponseCode::NotModified);
+ }
+ }
+ {
+ CompositeBuffer PartialChunkResult;
+ uint64_t PartialChunkModificationTag = 0;
+ {
+ CompositeBuffer ChunkResult;
+ HttpContentType ContentType;
+ CHECK(ProjectStore
+ .GetChunkRange("proj1"sv,
+ "oplog1"sv,
+ OidAsString(Attachments[OpIds[2]][1].first),
+ 5,
+ 1773,
+ HttpContentType::kCompressedBinary,
+ PartialChunkResult,
+ ContentType,
+ &PartialChunkModificationTag)
+ .first == HttpResponseCode::OK);
+ CHECK(PartialChunkResult);
+ IoHash PartialRawHash;
+ uint64_t PartialRawSize;
+ CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize);
+ CHECK(PartialRawSize >= 1773);
+
+ uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5);
+ SharedBuffer PartialDecompressed = PartialCompressedResult.Decompress(RawOffsetInPartialCompressed);
+ SharedBuffer FullDecompressed = Attachments[OpIds[2]][1].second.Decompress();
+ const uint8_t* FullDataPtr = &(reinterpret_cast<const uint8_t*>(FullDecompressed.GetView().GetData())[5]);
+ const uint8_t* PartialDataPtr = reinterpret_cast<const uint8_t*>(PartialDecompressed.GetView().GetData());
+ CHECK(FullDataPtr[0] == PartialDataPtr[0]);
+ }
+
+ {
+ CompositeBuffer ChunkResult;
+ HttpContentType ContentType;
+ CHECK(ProjectStore
+ .GetChunkRange("proj1"sv,
+ "oplog1"sv,
+ OidAsString(Attachments[OpIds[2]][1].first),
+ 5,
+ 1773,
+ HttpContentType::kCompressedBinary,
+ PartialChunkResult,
+ ContentType,
+ &PartialChunkModificationTag)
+ .first == HttpResponseCode::NotModified);
+ }
+ }
}
TEST_CASE("project.store.block")
@@ -6851,7 +8396,7 @@ TEST_CASE("project.store.iterateoplog")
TestOidData TestOids[NumTestOids];
for (const TestOidData& TestOid : TestOids)
{
- Oplog->AppendNewOplogEntry(CreateOplogPackage(TestOid.KeyAsOidNotOplogId, {}));
+ Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(TestOid.KeyAsOidNotOplogId, {}));
}
int Count = 0;
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 860f2c17d..4a33a367c 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -114,16 +114,18 @@ public:
std::optional<CbObject> GetOpByIndex(uint32_t Index);
std::optional<uint32_t> GetOpIndexByKey(const Oid& Key);
- IoBuffer FindChunk(const Oid& ChunkId);
+ IoBuffer FindChunk(const Oid& ChunkId, uint64_t* OptOutModificationTag);
IoBuffer GetChunkByRawHash(const IoHash& RawHash);
- bool IterateChunks(std::span<IoHash> RawHashes,
- const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
- WorkerThreadPool* OptionalWorkerPool,
- uint64_t LargeSizeLimit);
- bool IterateChunks(std::span<Oid> ChunkIds,
- const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback,
- WorkerThreadPool* OptionalWorkerPool,
- uint64_t LargeSizeLimit);
+ bool IterateChunks(std::span<IoHash> RawHashes,
+ bool IncludeModTag,
+ const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback,
+ WorkerThreadPool* OptionalWorkerPool,
+ uint64_t LargeSizeLimit);
+ bool IterateChunks(std::span<Oid> ChunkIds,
+ bool IncludeModTag,
+ const std::function<bool(size_t Index, const IoBuffer& Payload, uint64_t ModTag)>& AsyncCallback,
+ WorkerThreadPool* OptionalWorkerPool,
+ uint64_t LargeSizeLimit);
inline static const uint32_t kInvalidOp = ~0u;
/** Persist a new oplog entry
@@ -427,7 +429,8 @@ public:
uint64_t Size,
ZenContentType AcceptType,
CompositeBuffer& OutChunk,
- ZenContentType& OutContentType);
+ ZenContentType& OutContentType,
+ uint64_t* OptionalInOutModificationTag);
std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view ChunkId,
@@ -435,12 +438,14 @@ public:
uint64_t Size,
ZenContentType AcceptType,
CompositeBuffer& OutChunk,
- ZenContentType& OutContentType);
+ ZenContentType& OutContentType,
+ uint64_t* OptionalInOutModificationTag);
std::pair<HttpResponseCode, std::string> GetChunk(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view Cid,
ZenContentType AcceptType,
- IoBuffer& OutChunk);
+ IoBuffer& OutChunk,
+ uint64_t* OptionalInOutModificationTag);
std::pair<HttpResponseCode, std::string> PutChunk(const std::string_view ProjectId,
const std::string_view OplogId,
@@ -458,6 +463,11 @@ public:
const HttpServerRequest::QueryParams& Params,
CbObject& OutResponse);
+ std::pair<HttpResponseCode, std::string> GetChunks(const std::string_view ProjectId,
+ const std::string_view OplogId,
+ const CbObject& RequestObject,
+ CbPackage& OutResponsePackage);
+
bool Rpc(HttpServerRequest& HttpReq,
const std::string_view ProjectId,
const std::string_view OplogId,
diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp
index c0082b746..6b05442b3 100644
--- a/src/zenserver/projectstore/zenremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp
@@ -152,14 +152,22 @@ public:
{
CbObjectWriter RequestWriter;
RequestWriter.AddString("method"sv, "getchunks"sv);
- RequestWriter.BeginArray("chunks"sv);
+ RequestWriter.BeginObject("Request"sv);
{
- for (const IoHash& RawHash : RawHashes)
+ RequestWriter.BeginArray("Chunks"sv);
{
- RequestWriter.AddHash(RawHash);
+ for (const IoHash& RawHash : RawHashes)
+ {
+ RequestWriter.BeginObject();
+ {
+ RequestWriter.AddHash("RawHash", RawHash);
+ }
+ RequestWriter.EndObject();
+ }
}
+ RequestWriter.EndArray(); // "chunks"
}
- RequestWriter.EndArray(); // "chunks"
+ RequestWriter.EndObject();
Request = RequestWriter.Save();
}
diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp
index 6e14b7632..2bac6b756 100644
--- a/src/zenserver/vfs/vfsimpl.cpp
+++ b/src/zenserver/vfs/vfsimpl.cpp
@@ -47,7 +47,8 @@ VfsOplogDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t Byt
~0ull,
ZenContentType::kCompressedBinary,
/* out */ ChunkBuffer,
- /* out */ ContentType);
+ /* out */ ContentType,
+ /* OptionalInOutModificationTag */ nullptr);
if (Result.first == HttpResponseCode::OK)
{
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 9aa4ff613..50af7246e 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -329,31 +329,33 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes,
{
ZEN_MEMSCOPE(GetCasContainerTag());
- const size_t ChunkCount = ChunkHashes.size();
- if (ChunkCount < 3)
+ const size_t ChunkCount = ChunkHashes.size();
+ std::vector<size_t> FoundChunkIndexes;
+ std::vector<BlockStoreLocation> FoundChunkLocations;
+ FoundChunkIndexes.reserve(ChunkCount);
+ FoundChunkLocations.reserve(ChunkCount);
{
+ RwLock::SharedLockScope _(m_LocationMapLock);
for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ChunkIndex++)
{
- IoBuffer Chunk = FindChunk(ChunkHashes[ChunkIndex]);
- if (!AsyncCallback(ChunkIndex, Chunk))
+ if (auto KeyIt = m_LocationMap.find(ChunkHashes[ChunkIndex]); KeyIt != m_LocationMap.end())
{
- return false;
+ FoundChunkIndexes.push_back(ChunkIndex);
+ FoundChunkLocations.push_back(m_Locations[KeyIt->second].Get(m_PayloadAlignment));
}
}
- return true;
}
- std::vector<size_t> FoundChunkIndexes;
- std::vector<BlockStoreLocation> FoundChunkLocations;
- FoundChunkIndexes.reserve(ChunkCount);
- FoundChunkLocations.reserve(ChunkCount);
- RwLock::SharedLockScope _(m_LocationMapLock);
- for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ChunkIndex++)
+ if (FoundChunkLocations.size() < 3)
{
- if (auto KeyIt = m_LocationMap.find(ChunkHashes[ChunkIndex]); KeyIt != m_LocationMap.end())
+ for (size_t ChunkIndex : FoundChunkIndexes)
{
- FoundChunkIndexes.push_back(ChunkIndex);
- FoundChunkLocations.push_back(m_Locations[KeyIt->second].Get(m_PayloadAlignment));
+ IoBuffer Chunk = m_BlockStore.TryGetChunk(FoundChunkLocations[ChunkIndex]);
+ if (!AsyncCallback(ChunkIndex, Chunk))
+ {
+ return false;
+ }
}
+ return true;
}
auto DoOneBlock = [&](std::span<const size_t> ChunkIndexes) {