1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include <zencore/zencore.h>
#include <zenstore/basicfile.h>
#include <zenstore/caslog.h>
#include <zenstore/gc.h>
#include <atomic>
#include <limits>
#include <unordered_map>
namespace spdlog {
class logger;
}
namespace zen {
//////////////////////////////////////////////////////////////////////////
#pragma pack(push)
#pragma pack(1)
struct CasDiskLocation
{
// 20 bits blockindex
// 28 bits offset
// 32 bits size
CasDiskLocation(uint32_t BlockIndex, uint64_t Offset, uint64_t Size)
{
ZEN_ASSERT(BlockIndex < (1L << 20));
ZEN_ASSERT(Offset < (1L << 28));
ZEN_ASSERT(Size <= std::numeric_limits<std::uint32_t>::max());
m_Size = static_cast<uint32_t>(Size);
m_BlockIndexLowBits = static_cast<uint16_t>(BlockIndex & 0xffff);
m_OffsetLowBits = static_cast<uint16_t>(Offset & 0xffff);
m_BlockIndexAndOffsetHighBits = static_cast<uint16_t>(((BlockIndex >> 16) & 0xf) | (((Offset >> 16) & 0xfff) << 4));
}
CasDiskLocation() = default;
inline uint32_t GetBlockIndex() const
{
return (static_cast<uint32_t>(m_BlockIndexAndOffsetHighBits & 0xf) << 16) | m_BlockIndexLowBits;
}
inline uint64_t Offset() const { return (static_cast<uint64_t>(m_BlockIndexAndOffsetHighBits & 0xfff0) << 8) | m_OffsetLowBits; }
inline uint64_t Size() const { return m_Size; }
private:
uint32_t m_Size;
uint16_t m_BlockIndexLowBits;
uint16_t m_OffsetLowBits;
uint16_t m_BlockIndexAndOffsetHighBits;
#if 0
// 24 bits blockindex
// 24 bits offset
// 32 bits size
inline uint32_t GetBlockIndex() const
{
return (static_cast<uint32_t>(BlockIndexHighBits) << 16) | BlockIndexLowBits;
}
inline uint32_t Offset() const
{
return (static_cast<uint32_t>(OffsetHighBits) << 16) | OffsetLowBits;
}
uint32_t Size;
uint16_t BlockIndexLowBits;
uint16_t OffsetLowBits;
uint8_t BlockIndexHighBits;
uint8_t OffsetHighBits;
#endif
};
struct CasDiskIndexEntry
{
static const uint8_t kTombstone = 0x01;
IoHash Key;
CasDiskLocation Location;
ZenContentType ContentType = ZenContentType::kUnknownContentType;
uint8_t Flags = 0;
};
#pragma pack(pop)
static_assert(sizeof(CasDiskIndexEntry) == 32);
/** This implements a storage strategy for small CAS values
*
* New chunks are simply appended to a small object file, and an index is
* maintained to allow chunks to be looked up within the active small object
* files
*
*/
struct CasContainerStrategy final : public GcStorage
{
static constexpr uint32_t DefaultMaxBlockSize = 1u << 30; // 1 Gb
CasContainerStrategy(const CasStoreConfiguration& Config, CasGc& Gc);
~CasContainerStrategy();
CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash);
IoBuffer FindChunk(const IoHash& ChunkHash);
bool HaveChunk(const IoHash& ChunkHash);
void FilterChunks(CasChunkSet& InOutChunks);
void Initialize(const std::string_view ContainerBaseName, uint32_t MaxBlockSize, uint64_t Alignment, bool IsNewStore);
void Flush();
void Scrub(ScrubContext& Ctx);
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override { return {.DiskSize = m_TotalSize.load(std::memory_order::relaxed)}; }
private:
struct ChunkBlock;
void OpenContainer(bool IsNewStore);
spdlog::logger& Log() { return m_Log; }
const CasStoreConfiguration& m_Config;
spdlog::logger& m_Log;
uint64_t m_PayloadAlignment;
uint64_t m_MaxBlockSize;
bool m_IsInitialized = false;
TCasLogFile<CasDiskIndexEntry> m_CasLog;
std::string m_ContainerBaseName;
RwLock m_LocationMapLock;
std::unordered_map<IoHash, CasDiskLocation, IoHash::Hasher> m_LocationMap;
RwLock m_InsertLock; // used to serialize inserts
std::unordered_map<uint16_t, std::shared_ptr<ChunkBlock>> m_OpenBlocks;
std::weak_ptr<ChunkBlock> m_CurrentBlock;
uint16_t m_CurrentBlockIndex = 0;
std::atomic_uint32_t m_CurrentInsertOffset{};
std::atomic_uint64_t m_TotalSize{};
void MakeIndexSnapshot();
};
void compactcas_forcelink();
} // namespace zen
|