1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include <zencore/zencore.h>
#include <zenstore/basicfile.h>
#include <zenstore/caslog.h>
#include <zenstore/gc.h>
#include <atomic>
#include <limits>
#include <unordered_map>
namespace spdlog {
class logger;
}
namespace zen {
//////////////////////////////////////////////////////////////////////////
struct CasLocation
{
uint32_t BlockIndex;
uint64_t Offset;
uint64_t Size;
};
#pragma pack(push)
#pragma pack(1)
struct CasDiskLocation
{
constexpr static uint32_t MaxBlockIndexBits = 20;
constexpr static uint32_t MaxOffsetBits = 28;
constexpr static uint32_t MaxBlockIndex = (1ul << CasDiskLocation::MaxBlockIndexBits) - 1ul;
constexpr static uint32_t MaxOffset = (1ul << CasDiskLocation::MaxOffsetBits) - 1ul;
CasDiskLocation(const CasLocation& Location, uint64_t OffsetAlignement)
{
Init(Location.BlockIndex, Location.Offset / OffsetAlignement, Location.Size);
}
CasDiskLocation() = default;
inline CasLocation Get(uint64_t OffsetAlignement) const
{
return {.BlockIndex = BlockIndex(), .Offset = Offset() * OffsetAlignement, .Size = Size()};
}
private:
inline uint32_t BlockIndex() const { return (static_cast<uint32_t>(m_BlockIndexAndOffsetHighBits & 0xf) << 16) | m_BlockIndexLowBits; }
inline uint64_t Offset() const { return (static_cast<uint64_t>(m_BlockIndexAndOffsetHighBits & 0xfff0) << 12) | m_OffsetLowBits; }
inline uint64_t Size() const { return m_Size; }
inline void Init(uint32_t BlockIndex, uint64_t Offset, uint64_t Size)
{
ZEN_ASSERT(BlockIndex < (1L << 20));
ZEN_ASSERT(Offset < (1L << 28));
ZEN_ASSERT(Size <= std::numeric_limits<std::uint32_t>::max());
m_Size = static_cast<uint32_t>(Size);
m_BlockIndexLowBits = static_cast<uint16_t>(BlockIndex & 0xffff);
m_OffsetLowBits = static_cast<uint16_t>(Offset & 0xffff);
m_BlockIndexAndOffsetHighBits = static_cast<uint16_t>(((BlockIndex >> 16) & 0xf) | (((Offset >> 16) & 0xfff) << 4));
}
uint32_t m_Size;
uint16_t m_BlockIndexLowBits;
uint16_t m_OffsetLowBits;
uint16_t m_BlockIndexAndOffsetHighBits;
};
struct CasDiskIndexEntry
{
static const uint8_t kTombstone = 0x01;
IoHash Key;
CasDiskLocation Location;
ZenContentType ContentType = ZenContentType::kUnknownContentType;
uint8_t Flags = 0;
};
#pragma pack(pop)
static_assert(sizeof(CasDiskIndexEntry) == 32);
/** This implements a storage strategy for small CAS values
*
* New chunks are simply appended to a small object file, and an index is
* maintained to allow chunks to be looked up within the active small object
* files
*
*/
struct CasContainerStrategy final : public GcStorage
{
CasContainerStrategy(const CasStoreConfiguration& Config, CasGc& Gc);
~CasContainerStrategy();
CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash);
IoBuffer FindChunk(const IoHash& ChunkHash);
bool HaveChunk(const IoHash& ChunkHash);
void FilterChunks(CasChunkSet& InOutChunks);
void Initialize(const std::string_view ContainerBaseName, uint32_t MaxBlockSize, uint64_t Alignment, bool IsNewStore);
void Flush();
void Scrub(ScrubContext& Ctx);
virtual void CollectGarbage(GcContext& GcCtx) override;
virtual GcStorageSize StorageSize() const override { return {.DiskSize = m_TotalSize.load(std::memory_order::relaxed)}; }
private:
struct ChunkBlock;
void OpenContainer(bool IsNewStore);
spdlog::logger& Log() { return m_Log; }
const CasStoreConfiguration& m_Config;
spdlog::logger& m_Log;
uint64_t m_PayloadAlignment;
uint64_t m_MaxBlockSize;
bool m_IsInitialized = false;
TCasLogFile<CasDiskIndexEntry> m_CasLog;
std::string m_ContainerBaseName;
std::filesystem::path m_BlocksBasePath;
RwLock m_LocationMapLock;
std::unordered_map<IoHash, CasDiskLocation, IoHash::Hasher> m_LocationMap;
std::unordered_map<uint32_t, std::shared_ptr<ChunkBlock>> m_ChunkBlocks;
RwLock m_InsertLock; // used to serialize inserts
std::weak_ptr<ChunkBlock> m_WriteBlock;
std::uint64_t m_CurrentInsertOffset{};
std::atomic_uint32_t m_WriteBlockIndex{};
std::atomic_uint64_t m_TotalSize{};
void MakeIndexSnapshot();
};
void compactcas_forcelink();
} // namespace zen
|