1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
// Copyright Epic Games, Inc. All Rights Reserved.
#include "zenstore/scrubcontext.h"
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/string.h>
#include <zencore/workthreadpool.h>
namespace zen {
ScrubDeadlineExpiredException::ScrubDeadlineExpiredException() : std::runtime_error("scrubbing deadline expired")
{
}
ScrubDeadlineExpiredException::~ScrubDeadlineExpiredException()
{
}
//////////////////////////////////////////////////////////////////////////
ScrubContext::ScrubContext(WorkerThreadPool& InWorkerThreadPool, std::chrono::steady_clock::time_point Deadline)
: m_WorkerThreadPool(InWorkerThreadPool)
, m_Deadline(Deadline)
{
}
ScrubContext::~ScrubContext()
{
}
HashKeySet
ScrubContext::BadCids() const
{
RwLock::SharedLockScope _(m_Lock);
return m_BadCid;
}
bool
ScrubContext::IsBadCid(const IoHash& Cid) const
{
RwLock::SharedLockScope _(m_Lock);
return m_BadCid.ContainsHash(Cid);
}
void
ScrubContext::ReportBadCidChunks(std::span<IoHash> BadCasChunks)
{
RwLock::ExclusiveLockScope _(m_Lock);
m_BadCid.AddHashesToSet(BadCasChunks);
}
bool
ScrubContext::IsWithinDeadline() const
{
return std::chrono::steady_clock::now() < m_Deadline;
}
void
ScrubContext::ThrowIfDeadlineExpired() const
{
if (IsWithinDeadline())
return;
throw ScrubDeadlineExpiredException();
}
bool
ValidateCompressedBuffer(const CompositeBuffer& Buffer, const IoHash* OptionalExpectedHash)
{
IoHash HeaderRawHash;
uint64_t RawSize = 0;
uint64_t TotalCompressedSize = 0;
if (!CompressedBuffer::ValidateCompressedHeader(Buffer, HeaderRawHash, RawSize, &TotalCompressedSize))
{
if (OptionalExpectedHash)
{
ZEN_SCOPED_WARN("compressed buffer header validation failed for chunk with hash {}", *OptionalExpectedHash);
}
else
{
ZEN_SCOPED_WARN("compressed buffer header validation failed");
}
return false;
}
if (OptionalExpectedHash != nullptr && HeaderRawHash != (*OptionalExpectedHash))
{
ZEN_SCOPED_WARN("compressed buffer hash {} does not match expected hash {}", HeaderRawHash, *OptionalExpectedHash);
return false;
}
if (TotalCompressedSize != Buffer.GetSize())
{
ZEN_SCOPED_WARN("compressed buffer size does not match total compressed size in header for chunk {}", HeaderRawHash);
return false;
}
CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, /* out */ HeaderRawHash, /* out */ RawSize);
IoHashStream HashStream;
if (!Compressed.DecompressToStream(
0,
RawSize,
[&HashStream](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range) -> bool {
ZEN_UNUSED(SourceOffset, SourceSize, Offset);
for (const SharedBuffer& Segment : Range.GetSegments())
{
HashStream.Append(Segment);
}
return true;
}))
{
ZEN_SCOPED_WARN("compressed buffer could not be decompressed for chunk {}", HeaderRawHash);
return false;
}
IoHash DecompressedHash = HashStream.GetHash();
if (HeaderRawHash != DecompressedHash)
{
ZEN_SCOPED_WARN("decompressed hash {} differs from header hash {}", DecompressedHash, HeaderRawHash);
return false;
}
return true;
}
} // namespace zen
|