1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
// Copyright Epic Games, Inc. All Rights Reserved.
#include <zenstore/cas.h>
#include "CompactCas.h"
#include <zencore/except.h>
#include <zencore/memory.h>
#include <zencore/string.h>
#include <zencore/thread.h>
#include <zencore/uid.h>
#include <gsl/gsl-lite.hpp>
#include <functional>
struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive-
#include <atlfile.h>
#include <filesystem>
//////////////////////////////////////////////////////////////////////////
namespace zen {
void
CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore)
{
ZEN_ASSERT(IsPow2(Alignment));
ZEN_ASSERT(!m_IsInitialized);
m_PayloadAlignment = Alignment;
std::string BaseName(ContainerBaseName);
std::filesystem::path SobsPath = m_Config.RootDirectory / (BaseName + ".ucas");
std::filesystem::path SidxPath = m_Config.RootDirectory / (BaseName + ".uidx");
std::filesystem::path SlogPath = m_Config.RootDirectory / (BaseName + ".ulog");
m_SmallObjectFile.Open(SobsPath, IsNewStore);
m_SmallObjectIndex.Open(SidxPath, IsNewStore);
m_CasLog.Open(SlogPath, IsNewStore);
// TODO: should validate integrity of container files here
uint64_t MaxFileOffset = 0;
{
// This is not technically necessary but may help future static analysis
zen::RwLock::ExclusiveLockScope _(m_LocationMapLock);
m_CasLog.Replay([&](const CasDiskIndexEntry& Record) {
m_LocationMap[Record.Key] = Record.Location;
MaxFileOffset = std::max<uint64_t>(MaxFileOffset, Record.Location.Offset + Record.Location.Size);
});
}
m_CurrentInsertOffset = (MaxFileOffset + m_PayloadAlignment - 1) & ~(m_PayloadAlignment - 1);
m_CurrentIndexOffset = m_SmallObjectIndex.FileSize();
m_IsInitialized = true;
}
CasStore::InsertResult
CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash)
{
{
RwLock::SharedLockScope _(m_LocationMapLock);
auto KeyIt = m_LocationMap.find(ChunkHash);
if (KeyIt != m_LocationMap.end())
{
return CasStore::InsertResult{.New = false};
}
}
// New entry
RwLock::ExclusiveLockScope _(m_InsertLock);
const uint64_t InsertOffset = m_CurrentInsertOffset;
m_SmallObjectFile.Write(ChunkData, ChunkSize, InsertOffset);
m_CurrentInsertOffset = (m_CurrentInsertOffset + ChunkSize + m_PayloadAlignment - 1) & ~(m_PayloadAlignment - 1);
RwLock::ExclusiveLockScope __(m_LocationMapLock);
CasDiskLocation Location{.Offset = InsertOffset, .Size = /* TODO FIX */ uint32_t(ChunkSize)};
m_LocationMap[ChunkHash] = Location;
CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = Location};
m_CasLog.Append(IndexEntry);
return CasStore::InsertResult{.New = true};
}
CasStore::InsertResult
CasContainerStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
{
return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash);
}
IoBuffer
CasContainerStrategy::FindChunk(const IoHash& ChunkHash)
{
RwLock::SharedLockScope _(m_LocationMapLock);
auto KeyIt = m_LocationMap.find(ChunkHash);
if (KeyIt != m_LocationMap.end())
{
const CasDiskLocation& Location = KeyIt->second;
return zen::IoBufferBuilder::MakeFromFileHandle(m_SmallObjectFile.Handle(), Location.Offset, Location.Size);
}
// Not found
return IoBuffer();
}
void
CasContainerStrategy::Flush()
{
m_CasLog.Flush();
m_SmallObjectIndex.Flush();
m_SmallObjectFile.Flush();
}
} // namespace zen
|