1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include <zencore/zencore.h>
#include <zencore/iobuffer.h>
#include <zencore/iohash.h>
#include <zencore/string.h>
#include <zencore/thread.h>
#include <zencore/uid.h>
#include <zenstore/basicfile.h>
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
#include <zenstore/gc.h>
#if ZEN_PLATFORM_WINDOWS
# include <zencore/windows.h>
#endif
#include <atomic>
#include <unordered_map>
namespace spdlog {
class logger;
}
namespace zen {
//////////////////////////////////////////////////////////////////////////
#pragma pack(push)
#pragma pack(1)
struct CasDiskLocation
{
CasDiskLocation(uint64_t InOffset, uint64_t InSize)
{
ZEN_ASSERT(InOffset <= 0xff'ffff'ffff);
ZEN_ASSERT(InSize <= 0xff'ffff'ffff);
memcpy(&m_Offset[0], &InOffset, sizeof m_Offset);
memcpy(&m_Size[0], &InSize, sizeof m_Size);
}
CasDiskLocation() = default;
inline uint64_t GetOffset() const
{
uint64_t Offset = 0;
memcpy(&Offset, &m_Offset, sizeof m_Offset);
return Offset;
}
inline uint64_t GetSize() const
{
uint64_t Size = 0;
memcpy(&Size, &m_Size, sizeof m_Size);
return Size;
}
private:
uint8_t m_Offset[5];
uint8_t m_Size[5];
};
struct CasDiskIndexEntry
{
static const uint8_t kTombstone = 0x01;
IoHash Key;
CasDiskLocation Location;
ZenContentType ContentType = ZenContentType::kUnknownContentType;
uint8_t Flags = 0;
};
#pragma pack(pop)
static_assert(sizeof(CasDiskIndexEntry) == 32);
/** This implements a storage strategy for small CAS values
*
* New chunks are simply appended to a small object file, and an index is
* maintained to allow chunks to be looked up within the active small object
* files
*
*/
struct CasContainerStrategy final : public GcStorage
{
CasContainerStrategy(const CasStoreConfiguration& Config, CasGc& Gc);
~CasContainerStrategy();
CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash);
IoBuffer FindChunk(const IoHash& ChunkHash);
bool HaveChunk(const IoHash& ChunkHash);
void FilterChunks(CasChunkSet& InOutChunks);
void Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore);
void Flush();
void Scrub(ScrubContext& Ctx);
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override { return {.DiskSize = m_TotalSize.load(std::memory_order::relaxed)}; }
private:
void OpenContainer(bool IsNewStore);
void CloseContainer();
spdlog::logger& Log() { return m_Log; }
const CasStoreConfiguration& m_Config;
spdlog::logger& m_Log;
uint64_t m_PayloadAlignment = 1 << 4;
bool m_IsInitialized = false;
BasicFile m_SmallObjectFile;
BasicFile m_SmallObjectIndex;
TCasLogFile<CasDiskIndexEntry> m_CasLog;
std::string m_ContainerBaseName;
RwLock m_LocationMapLock;
std::unordered_map<IoHash, CasDiskLocation, IoHash::Hasher> m_LocationMap;
RwLock m_InsertLock; // used to serialize inserts
std::atomic_uint64_t m_CurrentInsertOffset{};
std::atomic_uint64_t m_CurrentIndexOffset{};
std::atomic_uint64_t m_TotalSize{};
void MakeSnapshot();
};
void compactcas_forcelink();
} // namespace zen
|