1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include <zencore/jobqueue.h>
#include "projectstore.h"
#include <zenutil/chunkblock.h>
#include <unordered_set>
namespace zen {
class CidStore;
class WorkerThreadPool;
struct ChunkedInfo;
class RemoteProjectStore
{
public:
struct Result
{
int32_t ErrorCode{};
double ElapsedSeconds{};
std::string Reason;
std::string Text;
};
struct CreateContainerResult : public Result
{
};
struct SaveResult : public Result
{
std::unordered_set<IoHash, IoHash::Hasher> Needs;
IoHash RawHash;
};
struct FinalizeResult : public Result
{
std::unordered_set<IoHash, IoHash::Hasher> Needs;
};
struct SaveAttachmentResult : public Result
{
};
struct SaveAttachmentsResult : public Result
{
};
struct LoadAttachmentResult : public Result
{
IoBuffer Bytes;
};
struct LoadContainerResult : public Result
{
CbObject ContainerObject;
};
struct LoadAttachmentsResult : public Result
{
std::vector<std::pair<IoHash, CompressedBuffer>> Chunks;
};
struct GetKnownBlocksResult : public Result
{
std::vector<ThinChunkBlockDescription> Blocks;
};
struct RemoteStoreInfo
{
bool CreateBlocks;
bool UseTempBlockFiles;
bool AllowChunking;
std::string ContainerName;
std::string Description;
};
struct Stats
{
std::uint64_t m_SentBytes;
std::uint64_t m_ReceivedBytes;
std::uint64_t m_RequestTimeNS;
std::uint64_t m_RequestCount;
std::uint64_t m_PeakSentBytes;
std::uint64_t m_PeakReceivedBytes;
std::uint64_t m_PeakBytesPerSec;
};
RemoteProjectStore();
virtual ~RemoteProjectStore();
virtual RemoteStoreInfo GetInfo() const = 0;
virtual Stats GetStats() const = 0;
virtual CreateContainerResult CreateContainer() = 0;
virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0;
virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&& Block) = 0;
virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0;
virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads) = 0;
virtual LoadContainerResult LoadContainer() = 0;
virtual GetKnownBlocksResult GetKnownBlocks() = 0;
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0;
virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0;
};
struct RemoteStoreOptions
{
static const size_t DefaultMaxBlockSize = 64u * 1024u * 1024u;
static const size_t DefaultMaxChunkEmbedSize = 3u * 512u * 1024u;
static const size_t DefaultChunkFileSizeLimit = 256u * 1024u * 1024u;
size_t MaxBlockSize = DefaultMaxBlockSize;
size_t MaxChunkEmbedSize = DefaultMaxChunkEmbedSize;
size_t ChunkFileSizeLimit = DefaultChunkFileSizeLimit;
};
typedef std::function<IoBuffer(const IoHash& AttachmentHash)> TGetAttachmentBufferFunc;
RemoteProjectStore::LoadContainerResult BuildContainer(
CidStore& ChunkStore,
ProjectStore::Project& Project,
ProjectStore::Oplog& Oplog,
size_t MaxBlockSize,
size_t MaxChunkEmbedSize,
size_t ChunkFileSizeLimit,
bool BuildBlocks,
bool IgnoreMissingAttachments,
bool AllowChunking,
const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock,
const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment,
const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks,
bool EmbedLooseFiles);
class JobContext;
RemoteProjectStore::Result SaveOplogContainer(ProjectStore::Oplog& Oplog,
const CbObject& ContainerObject,
const std::function<void(std::span<IoHash> RawHashes)>& OnReferencedAttachments,
const std::function<bool(const IoHash& RawHash)>& HasAttachment,
const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock,
const std::function<void(const IoHash& RawHash)>& OnNeedAttachment,
const std::function<void(const ChunkedInfo& Chunked)>& OnChunkedAttachment,
JobContext* OptionalContext);
RemoteProjectStore::Result SaveOplog(CidStore& ChunkStore,
RemoteProjectStore& RemoteStore,
ProjectStore::Project& Project,
ProjectStore::Oplog& Oplog,
size_t MaxBlockSize,
size_t MaxChunkEmbedSize,
size_t ChunkFileSizeLimit,
bool EmbedLooseFiles,
bool ForceUpload,
bool IgnoreMissingAttachments,
JobContext* OptionalContext);
RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore,
RemoteProjectStore& RemoteStore,
ProjectStore::Oplog& Oplog,
bool ForceDownload,
bool IgnoreMissingAttachments,
bool CleanOplog,
JobContext* OptionalContext);
std::vector<IoHash> GetBlockHashesFromOplog(CbObjectView ContainerObject);
std::vector<ThinChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes);
} // namespace zen
|