1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include <zencore/jobqueue.h>
#include "projectstore.h"
#include <unordered_set>
namespace zen {
class CidStore;
class WorkerThreadPool;
class RemoteProjectStore
{
public:
struct Result
{
int32_t ErrorCode{};
double ElapsedSeconds{};
std::string Reason;
std::string Text;
};
struct SaveResult : public Result
{
std::unordered_set<IoHash, IoHash::Hasher> Needs;
IoHash RawHash;
};
struct FinalizeResult : public Result
{
std::unordered_set<IoHash, IoHash::Hasher> Needs;
};
struct SaveAttachmentResult : public Result
{
};
struct SaveAttachmentsResult : public Result
{
};
struct LoadAttachmentResult : public Result
{
IoBuffer Bytes;
};
struct LoadContainerResult : public Result
{
CbObject ContainerObject;
};
struct LoadAttachmentsResult : public Result
{
std::vector<std::pair<IoHash, CompressedBuffer>> Chunks;
};
struct RemoteStoreInfo
{
bool CreateBlocks;
bool UseTempBlockFiles;
std::string Description;
};
RemoteProjectStore();
virtual ~RemoteProjectStore();
virtual RemoteStoreInfo GetInfo() const = 0;
virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0;
virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) = 0;
virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0;
virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads) = 0;
virtual LoadContainerResult LoadContainer() = 0;
virtual LoadContainerResult LoadBaseContainer() = 0;
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0;
virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0;
};
struct RemoteStoreOptions
{
size_t MaxBlockSize = 128u * 1024u * 1024u;
size_t MaxChunkEmbedSize = 1024u * 1024u;
};
RemoteProjectStore::LoadContainerResult BuildContainer(
CidStore& ChunkStore,
ProjectStore::Project& Project,
ProjectStore::Oplog& Oplog,
size_t MaxBlockSize,
size_t MaxChunkEmbedSize,
bool BuildBlocks,
const std::function<void(CompressedBuffer&&, const IoHash&)>& AsyncOnBlock,
const std::function<void(const IoHash&)>& OnLargeAttachment,
const std::function<void(const std::unordered_set<IoHash, IoHash::Hasher>)>& OnBlockChunks,
tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher>*
OutOptionalTempAttachments); // Set OutOptionalTempAttachments to nullptr to avoid embedding loose "additional files"
class JobContext;
RemoteProjectStore::Result SaveOplogContainer(ProjectStore::Oplog& Oplog,
const CbObject& ContainerObject,
const std::function<bool(const IoHash& RawHash)>& HasAttachment,
const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock,
const std::function<void(const IoHash& RawHash)>& OnNeedAttachment,
JobContext* OptionalContext);
RemoteProjectStore::Result SaveOplog(CidStore& ChunkStore,
RemoteProjectStore& RemoteStore,
ProjectStore::Project& Project,
ProjectStore::Oplog& Oplog,
size_t MaxBlockSize,
size_t MaxChunkEmbedSize,
bool EmbedLooseFiles,
bool BuildBlocks,
bool UseTempBlocks,
bool ForceUpload,
JobContext* OptionalContext);
RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore,
RemoteProjectStore& RemoteStore,
ProjectStore::Oplog& Oplog,
bool ForceDownload,
JobContext* OptionalContext);
CompressedBuffer GenerateBlock(std::vector<SharedBuffer>&& Chunks);
bool IterateBlock(IoBuffer&& CompressedBlock, std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor);
} // namespace zen
|