aboutsummaryrefslogtreecommitdiff
path: root/zenstore
diff options
context:
space:
mode:
Diffstat (limited to 'zenstore')
-rw-r--r--zenstore/basicfile.cpp86
-rw-r--r--zenstore/caslog.cpp83
-rw-r--r--zenstore/cidstore.cpp85
-rw-r--r--zenstore/compactcas.h5
-rw-r--r--zenstore/include/zenstore/CAS.h2
-rw-r--r--zenstore/include/zenstore/basicfile.h34
-rw-r--r--zenstore/include/zenstore/caslog.h28
-rw-r--r--zenstore/include/zenstore/cidstore.h38
-rw-r--r--zenstore/zenstore.vcxproj4
-rw-r--r--zenstore/zenstore.vcxproj.filters8
10 files changed, 271 insertions, 102 deletions
diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp
new file mode 100644
index 000000000..7f10fc5e6
--- /dev/null
+++ b/zenstore/basicfile.cpp
@@ -0,0 +1,86 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenstore/basicfile.h"
+
+#include <zencore/except.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+
+#include <fmt/format.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+using namespace fmt::literals;
+
+void
+BasicFile::Open(std::filesystem::path FileName, bool isCreate)
+{
+ const DWORD dwCreationDisposition = isCreate ? CREATE_ALWAYS : OPEN_EXISTING;
+
+ HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition);
+
+ if (FAILED(hRes))
+ {
+ ThrowSystemException(hRes, "Failed to open bucket sobs file '{}'"_format(FileName));
+ }
+}
+
+void
+BasicFile::Read(void* Data, uint64_t Size, uint64_t Offset)
+{
+ OVERLAPPED Ovl{};
+
+ Ovl.Offset = DWORD(Offset & 0xffff'ffffu);
+ Ovl.OffsetHigh = DWORD(Offset >> 32);
+
+ HRESULT hRes = m_File.Read(Data, gsl::narrow<DWORD>(Size), &Ovl);
+
+ if (FAILED(hRes))
+ {
+ ThrowSystemException(hRes, "Failed to read from file '{}'"_format(zen::PathFromHandle(m_File)));
+ }
+}
+
+IoBuffer
+BasicFile::ReadAll()
+{
+ IoBuffer Buffer(FileSize());
+
+ Read((void*)Buffer.Data(), Buffer.Size(), 0);
+
+ return Buffer;
+}
+
+void
+BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
+{
+ OVERLAPPED Ovl{};
+
+ Ovl.Offset = DWORD(Offset & 0xffff'ffffu);
+ Ovl.OffsetHigh = DWORD(Offset >> 32);
+
+ HRESULT hRes = m_File.Write(Data, gsl::narrow<DWORD>(Size), &Ovl);
+
+ if (FAILED(hRes))
+ {
+ ThrowSystemException(hRes, "Failed to write to file '{}'"_format(zen::PathFromHandle(m_File)));
+ }
+}
+
+void
+BasicFile::Flush()
+{
+ m_File.Flush();
+}
+
+uint64_t
+BasicFile::FileSize()
+{
+ ULONGLONG Sz;
+ m_File.GetSize(Sz);
+
+ return uint64_t(Sz);
+}
+
+} // namespace zen
diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp
index c648cea5e..0ef3ed1bd 100644
--- a/zenstore/caslog.cpp
+++ b/zenstore/caslog.cpp
@@ -125,7 +125,10 @@ CasLogFile::Replay(std::function<void(const void*)>&& Handler)
m_File.Seek(LogBaseOffset, FILE_BEGIN);
HRESULT hRes = m_File.Read(ReadBuffer.data(), gsl::narrow<DWORD>(LogDataSize));
- zen::ThrowIfFailed(hRes, "Failed to read log file");
+ if (FAILED(hRes))
+ {
+ zen::ThrowSystemException(hRes, "Failed to read log file");
+ }
for (int i = 0; i < LogEntryCount; ++i)
{
@@ -140,9 +143,7 @@ CasLogFile::Append(const void* DataPointer, uint64_t DataSize)
if (FAILED(hRes))
{
- throw std::system_error(GetLastError(),
- std::system_category(),
- "Failed to write to log file '{}'"_format(zen::PathFromHandle(m_File)));
+ zen::ThrowSystemException(hRes, "Failed to write to log file '{}'"_format(zen::PathFromHandle(m_File)));
}
}
@@ -152,78 +153,4 @@ CasLogFile::Flush()
m_File.Flush();
}
-//////////////////////////////////////////////////////////////////////////
-
-void
-CasBlobFile::Open(std::filesystem::path FileName, bool isCreate)
-{
- const DWORD dwCreationDisposition = isCreate ? CREATE_ALWAYS : OPEN_EXISTING;
-
- HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition);
-
- if (FAILED(hRes))
- {
- throw std::system_error(GetLastError(), std::system_category(), "Failed to open bucket sobs file '{}'"_format(FileName));
- }
-}
-
-void
-CasBlobFile::Read(void* Data, uint64_t Size, uint64_t Offset)
-{
- OVERLAPPED Ovl{};
-
- Ovl.Offset = DWORD(Offset & 0xffff'ffffu);
- Ovl.OffsetHigh = DWORD(Offset >> 32);
-
- HRESULT hRes = m_File.Read(Data, gsl::narrow<DWORD>(Size), &Ovl);
-
- if (FAILED(hRes))
- {
- throw std::system_error(GetLastError(),
- std::system_category(),
- "Failed to read from file '{}'"_format(zen::PathFromHandle(m_File)));
- }
-}
-
-IoBuffer
-CasBlobFile::ReadAll()
-{
- IoBuffer Buffer(FileSize());
-
- Read((void*)Buffer.Data(), Buffer.Size(), 0);
-
- return Buffer;
-}
-
-void
-CasBlobFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
-{
- OVERLAPPED Ovl{};
-
- Ovl.Offset = DWORD(Offset & 0xffff'ffffu);
- Ovl.OffsetHigh = DWORD(Offset >> 32);
-
- HRESULT hRes = m_File.Write(Data, gsl::narrow<DWORD>(Size), &Ovl);
-
- if (FAILED(hRes))
- {
- throw std::system_error(GetLastError(), std::system_category(), "Failed to write to file '{}'"_format(zen::PathFromHandle(m_File)));
- }
-}
-
-void
-CasBlobFile::Flush()
-{
- m_File.Flush();
-}
-
-uint64_t
-CasBlobFile::FileSize()
-{
- ULONGLONG Sz;
- m_File.GetSize(Sz);
-
- return uint64_t(Sz);
-}
-
} // namespace zen
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
new file mode 100644
index 000000000..e041574a4
--- /dev/null
+++ b/zenstore/cidstore.cpp
@@ -0,0 +1,85 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenstore/cidstore.h"
+
+#include <zencore/filesystem.h>
+#include <zenstore/CAS.h>
+#include <zenstore/caslog.h>
+
+#include <spdlog/spdlog.h>
+#include <filesystem>
+
+namespace zen {
+
+struct CidStore::CidState
+{
+ CidState(CasStore& InCasStore) : m_CasStore(InCasStore) {}
+
+ struct IndexEntry
+ {
+ IoHash Uncompressed;
+ IoHash Compressed;
+ };
+
+ CasStore& m_CasStore;
+ TCasLogFile<IndexEntry> m_LogFile;
+
+ RwLock m_Lock;
+ tsl::robin_map<IoHash, IoHash> m_CidMap;
+
+ void AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
+ {
+ RwLock::ExclusiveLockScope _(m_Lock);
+ m_CidMap.insert_or_assign(DecompressedId, Compressed);
+ m_LogFile.Append({.Uncompressed = DecompressedId, .Compressed = Compressed});
+ }
+
+ IoBuffer FindChunkByCid(const IoHash& DecompressedId)
+ {
+ if (auto It = m_CidMap.find(DecompressedId); It != m_CidMap.end())
+ {
+ return m_CasStore.FindChunk(It->second);
+ }
+
+ return IoBuffer();
+ }
+
+ void InitializeIndex(const std::filesystem::path& RootDir)
+ {
+ zen::CreateDirectories(RootDir);
+ std::filesystem::path SlogPath{RootDir / "cid.slog"};
+
+ bool IsNew = !std::filesystem::exists(SlogPath);
+
+ m_LogFile.Open(SlogPath, IsNew);
+
+ m_LogFile.Replay([&](const IndexEntry& Ie) { m_CidMap.insert_or_assign(Ie.Uncompressed, Ie.Compressed); });
+
+ spdlog::debug("CID index initialized: {} entries found", m_CidMap.size());
+ }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+CidStore::CidStore(CasStore& InCasStore, const std::filesystem::path& RootDir) : m_Impl(std::make_unique<CidState>(InCasStore))
+{
+ m_Impl->InitializeIndex(RootDir);
+}
+
+CidStore::~CidStore()
+{
+}
+
+void
+CidStore::AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
+{
+ m_Impl->AddCompressedCid(DecompressedId, Compressed);
+}
+
+IoBuffer
+CidStore::FindChunkByCid(const IoHash& DecompressedId)
+{
+ return m_Impl->FindChunkByCid(DecompressedId);
+}
+
+} // namespace zen
diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h
index 4d318c2e2..db115c85d 100644
--- a/zenstore/compactcas.h
+++ b/zenstore/compactcas.h
@@ -10,6 +10,7 @@
#include <zencore/thread.h>
#include <zencore/uid.h>
#include <zencore/windows.h>
+#include <zenstore/basicfile.h>
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
@@ -52,8 +53,8 @@ private:
CasStore::Stats& m_Stats;
uint64_t m_PayloadAlignment = 1 << 4;
bool m_IsInitialized = false;
- CasBlobFile m_SmallObjectFile;
- CasBlobFile m_SmallObjectIndex;
+ BasicFile m_SmallObjectFile;
+ BasicFile m_SmallObjectIndex;
TCasLogFile<CasDiskIndexEntry> m_CasLog;
RwLock m_LocationMapLock;
diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h
index 8b9a66e3f..9c569f2aa 100644
--- a/zenstore/include/zenstore/CAS.h
+++ b/zenstore/include/zenstore/CAS.h
@@ -23,7 +23,7 @@ struct CasStoreConfiguration
// Threshold below which values are considered 'tiny' and managed using the 'tiny values' strategy
uint64_t TinyValueThreshold = 1024;
- // Threshold above which values are considered 'tiny' and managed using the 'huge values' strategy
+ // Threshold above which values are considered 'huge' and managed using the 'huge values' strategy
uint64_t HugeValueThreshold = 1024 * 1024;
};
diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h
new file mode 100644
index 000000000..b38feb3da
--- /dev/null
+++ b/zenstore/include/zenstore/basicfile.h
@@ -0,0 +1,34 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iobuffer.h>
+#include <zencore/zencore.h>
+
+#include <zencore/windows.h>
+
+#include <atlfile.h>
+#include <filesystem>
+
+namespace zen {
+
+/**
+ * Probably the most basic file abstraction in the universe
+ */
+
+class BasicFile
+{
+public:
+ void Open(std::filesystem::path FileName, bool IsCreate);
+ void Read(void* Data, uint64_t Size, uint64_t Offset);
+ void Write(const void* Data, uint64_t Size, uint64_t Offset);
+ void Flush();
+ uint64_t FileSize();
+ void* Handle() { return m_File; }
+ IoBuffer ReadAll();
+
+private:
+ CAtlFile m_File;
+};
+
+} // namespace zen
diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h
index b318577d7..aea855e4c 100644
--- a/zenstore/include/zenstore/caslog.h
+++ b/zenstore/include/zenstore/caslog.h
@@ -69,28 +69,14 @@ public:
});
}
- void Append(const T& Record) { CasLogFile::Append(&Record, sizeof Record); }
- void Open(std::filesystem::path FileName, bool IsCreate) { CasLogFile::Open(FileName, sizeof(T), IsCreate); }
-};
-
-//////////////////////////////////////////////////////////////////////////
-//
-// This should go in its own header
-//
-
-class CasBlobFile
-{
-public:
- void Open(std::filesystem::path FileName, bool IsCreate);
- void Read(void* Data, uint64_t Size, uint64_t Offset);
- void Write(const void* Data, uint64_t Size, uint64_t Offset);
- void Flush();
- uint64_t FileSize();
- void* Handle() { return m_File; }
- IoBuffer ReadAll();
+ void Append(const T& Record)
+ {
+ // TODO: implement some more efficent path here so we don't end up with
+ // a syscall per append
-private:
- CAtlFile m_File;
+ CasLogFile::Append(&Record, sizeof Record);
+ }
+ void Open(std::filesystem::path FileName, bool IsCreate) { CasLogFile::Open(FileName, sizeof(T), IsCreate); }
};
} // namespace zen
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
new file mode 100644
index 000000000..e365b198e
--- /dev/null
+++ b/zenstore/include/zenstore/cidstore.h
@@ -0,0 +1,38 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <tsl/robin_map.h>
+#include <zencore/iohash.h>
+
+namespace std::filesystem {
+class path;
+}
+
+namespace zen {
+
+class CasStore;
+class IoBuffer;
+
+/** Content Store
+ *
+ * Data in the content store is referenced by content identifiers (CIDs), rather than their
+ * literal hash. This is required to map uncompressed hashes to compressed hashes and may
+ * be used to deal with other indirections in the future.
+ *
+ */
+class CidStore
+{
+public:
+ CidStore(CasStore& InCasStore, const std::filesystem::path& RootDir);
+ ~CidStore();
+
+ void AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed);
+ IoBuffer FindChunkByCid(const IoHash& DecompressedId);
+
+private:
+ struct CidState;
+ std::unique_ptr<CidState> m_Impl;
+};
+
+} // namespace zen
diff --git a/zenstore/zenstore.vcxproj b/zenstore/zenstore.vcxproj
index 06cb9db32..0e4caa92c 100644
--- a/zenstore/zenstore.vcxproj
+++ b/zenstore/zenstore.vcxproj
@@ -11,8 +11,10 @@
</ProjectConfiguration>
</ItemGroup>
<ItemGroup>
+ <ClCompile Include="basicfile.cpp" />
<ClCompile Include="CAS.cpp" />
<ClCompile Include="caslog.cpp" />
+ <ClCompile Include="cidstore.cpp" />
<ClCompile Include="compactcas.cpp" />
<ClCompile Include="filecas.cpp" />
<ClCompile Include="gc.cpp" />
@@ -21,6 +23,8 @@
<ItemGroup>
<ClInclude Include="compactcas.h" />
<ClInclude Include="filecas.h" />
+ <ClInclude Include="include\zenstore\basicfile.h" />
+ <ClInclude Include="include\zenstore\cidstore.h" />
<ClInclude Include="include\zenstore\gc.h" />
<ClInclude Include="include\zenstore\scrub.h" />
<ClInclude Include="include\zenstore\CAS.h" />
diff --git a/zenstore/zenstore.vcxproj.filters b/zenstore/zenstore.vcxproj.filters
index 6ab5a7cb2..3dfb89dbf 100644
--- a/zenstore/zenstore.vcxproj.filters
+++ b/zenstore/zenstore.vcxproj.filters
@@ -5,11 +5,19 @@
<ClCompile Include="caslog.cpp" />
<ClCompile Include="compactcas.cpp" />
<ClCompile Include="filecas.cpp" />
+ <ClCompile Include="gc.cpp" />
+ <ClCompile Include="scrub.cpp" />
+ <ClCompile Include="basicfile.cpp" />
+ <ClCompile Include="cidstore.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="compactcas.h" />
<ClInclude Include="filecas.h" />
<ClInclude Include="include\zenstore\CAS.h" />
<ClInclude Include="include\zenstore\caslog.h" />
+ <ClInclude Include="include\zenstore\gc.h" />
+ <ClInclude Include="include\zenstore\scrub.h" />
+ <ClInclude Include="include\zenstore\basicfile.h" />
+ <ClInclude Include="include\zenstore\cidstore.h" />
</ItemGroup>
</Project> \ No newline at end of file
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-21 12:51:01 +0000