1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include <zencore/jobqueue.h>
#include "projectstore.h"
#include <unordered_set>
namespace zen {
class CidStore;
class WorkerThreadPool;
struct ChunkedInfo;
class RemoteProjectStore
{
public:
struct Result
{
int32_t ErrorCode{};
double ElapsedSeconds{};
std::string Reason;
std::string Text;
};
struct SaveResult : public Result
{
std::unordered_set<IoHash, IoHash::Hasher> Needs;
IoHash RawHash;
};
struct FinalizeResult : public Result
{
std::unordered_set<IoHash, IoHash::Hasher> Needs;
};
struct SaveAttachmentResult : public Result
{
};
struct SaveAttachmentsResult : public Result
{
};
struct HasAttachmentsResult : public Result
{
std::unordered_set<IoHash, IoHash::Hasher> Needs;
};
struct LoadAttachmentResult : public Result
{
IoBuffer Bytes;
};
struct LoadContainerResult : public Result
{
CbObject ContainerObject;
};
struct LoadAttachmentsResult : public Result
{
std::vector<std::pair<IoHash, CompressedBuffer>> Chunks;
};
struct RemoteStoreInfo
{
bool CreateBlocks;
bool UseTempBlockFiles;
std::string ContainerName;
std::string BaseContainerName;
std::string Description;
};
RemoteProjectStore();
virtual ~RemoteProjectStore();
virtual RemoteStoreInfo GetInfo() const = 0;
virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0;
virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) = 0;
virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0;
virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads) = 0;
virtual LoadContainerResult LoadContainer() = 0;
virtual LoadContainerResult LoadBaseContainer() = 0;
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0;
virtual HasAttachmentsResult HasAttachments(const std::span<IoHash> RawHashes) = 0;
virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0;
};
struct RemoteStoreOptions
{
static const size_t DefaultMaxBlockSize = 64u * 1024u * 1024u;
static const size_t DefaultMaxChunkEmbedSize = 3u * 512u * 1024u;
static const size_t DefaultChunkFileSizeLimit = 256u * 1024u * 1024u;
size_t MaxBlockSize = DefaultMaxBlockSize;
size_t MaxChunkEmbedSize = DefaultMaxChunkEmbedSize;
size_t ChunkFileSizeLimit = DefaultChunkFileSizeLimit;
};
typedef std::function<IoBuffer(const IoHash& AttachmentHash)> TGetAttachmentBufferFunc;
typedef std::function<CompositeBuffer(const IoHash& RawHash)> FetchChunkFunc;
RemoteProjectStore::LoadContainerResult BuildContainer(
CidStore& ChunkStore,
ProjectStore::Project& Project,
ProjectStore::Oplog& Oplog,
size_t MaxBlockSize,
size_t MaxChunkEmbedSize,
size_t ChunkFileSizeLimit,
bool BuildBlocks,
bool IgnoreMissingAttachments,
const std::function<void(CompressedBuffer&&, const IoHash&)>& AsyncOnBlock,
const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment,
const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks,
bool EmbedLooseFiles);
class JobContext;
RemoteProjectStore::Result SaveOplogContainer(ProjectStore::Oplog& Oplog,
const CbObject& ContainerObject,
const std::function<bool(const IoHash& RawHash)>& HasAttachment,
const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock,
const std::function<void(const IoHash& RawHash)>& OnNeedAttachment,
const std::function<void(const ChunkedInfo& Chunked)>& OnChunkedAttachment,
JobContext* OptionalContext);
RemoteProjectStore::Result SaveOplog(CidStore& ChunkStore,
RemoteProjectStore& RemoteStore,
ProjectStore::Project& Project,
ProjectStore::Oplog& Oplog,
size_t MaxBlockSize,
size_t MaxChunkEmbedSize,
size_t ChunkFileSizeLimit,
bool EmbedLooseFiles,
bool ForceUpload,
bool IgnoreMissingAttachments,
JobContext* OptionalContext);
RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore,
RemoteProjectStore& RemoteStore,
ProjectStore::Oplog& Oplog,
bool ForceDownload,
bool IgnoreMissingAttachments,
JobContext* OptionalContext);
CompressedBuffer GenerateBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks);
bool IterateBlock(IoBuffer&& CompressedBlock, std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor);
} // namespace zen
|