aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMartin Ridgers <[email protected]>2021-10-11 10:32:13 +0200
committerMartin Ridgers <[email protected]>2021-10-11 10:32:13 +0200
commit735b478c1425cc9a1e407bf917dc8daa4ab16b11 (patch)
tree6e253ee94a5db9244756571084f3f07472e4e2b3
parentFixed undefined mimalloc symbols (diff)
parentiobuffer: Changed MakeFromTemporaryFile so it accepts a path instead of a cha... (diff)
downloadzen-735b478c1425cc9a1e407bf917dc8daa4ab16b11.tar.xz
zen-735b478c1425cc9a1e407bf917dc8daa4ab16b11.zip
Merged main
-rw-r--r--zencore/filesystem.cpp4
-rw-r--r--zencore/include/zencore/iobuffer.h2
-rw-r--r--zencore/include/zencore/testutils.h1
-rw-r--r--zencore/iobuffer.cpp6
-rw-r--r--zencore/string.cpp12
-rw-r--r--zencore/testutils.cpp7
-rw-r--r--zenserver/projectstore.cpp2
-rw-r--r--zenserver/vfs.cpp900
-rw-r--r--zenserver/vfs.h36
-rw-r--r--zenserver/zenserver.vcxproj5
-rw-r--r--zenserver/zenserver.vcxproj.filters2
-rw-r--r--zenstore/basicfile.cpp12
-rw-r--r--zenstore/filecas.cpp145
-rw-r--r--zenstore/filecas.h2
-rw-r--r--zenstore/include/zenstore/CAS.h2
-rw-r--r--zenstore/include/zenstore/basicfile.h2
-rw-r--r--zenstore/zenstore.cpp2
17 files changed, 185 insertions, 957 deletions
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index 9936f30ec..c3edf656e 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -842,9 +842,9 @@ TEST_CASE("filesystem")
{
using namespace std::filesystem;
- // GetExePath
+ // GetExePath -- this is not a great test as it's so dependent on where the this code gets linked in
path BinPath = GetRunningExecutablePath();
- const bool ExpectedExe = BinPath.stem() == "zencore-test" || BinPath.stem() == "zenserver-test";
+ const bool ExpectedExe = BinPath.stem() == "zencore-test" || BinPath.stem() == "zenserver-test" || BinPath.stem() == "zenstore-test";
CHECK(ExpectedExe);
CHECK(is_regular_file(BinPath));
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 110cd7d9d..5fbeaeaeb 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -372,7 +372,7 @@ class IoBufferBuilder
public:
ZENCORE_API static IoBuffer MakeFromFile(const path_char_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
- ZENCORE_API static IoBuffer MakeFromTemporaryFile(const path_char_t* FileName);
+ ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName);
ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull);
inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
};
diff --git a/zencore/include/zencore/testutils.h b/zencore/include/zencore/testutils.h
index 72d985d5c..04648c6de 100644
--- a/zencore/include/zencore/testutils.h
+++ b/zencore/include/zencore/testutils.h
@@ -11,6 +11,7 @@ std::filesystem::path CreateTemporaryDirectory();
class ScopedTemporaryDirectory
{
public:
+ explicit ScopedTemporaryDirectory(std::filesystem::path Directory);
ScopedTemporaryDirectory();
~ScopedTemporaryDirectory();
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index 04685defc..922c14f83 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -437,7 +437,7 @@ IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint
}
IoBuffer
-IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName)
+IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName)
{
uint64_t FileSize;
void* Handle;
@@ -449,7 +449,7 @@ IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName)
// when a file has been written to a staging directory, and is going
// to be moved in place
- HRESULT hRes = DataFile.Create(FileName, GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING);
+ HRESULT hRes = DataFile.Create(FileName.native().c_str(), GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING);
if (FAILED(hRes))
{
@@ -460,7 +460,7 @@ IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName)
Handle = DataFile.Detach();
# else
- int Fd = open(FileName, O_RDONLY);
+ int Fd = open(FileName.native().c_str(), O_RDONLY);
if (Fd < 0)
{
return {};
diff --git a/zencore/string.cpp b/zencore/string.cpp
index 4bf0378b2..efa84ea73 100644
--- a/zencore/string.cpp
+++ b/zencore/string.cpp
@@ -341,7 +341,17 @@ NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer)
size_t
NiceByteRateToBuffer(uint64_t Num, uint64_t ElapsedMs, std::span<char> Buffer)
{
- size_t n = NiceNumGeneral(Num * 1000 / ElapsedMs, Buffer, kNicenumBytes);
+ size_t n = 0;
+
+ if (ElapsedMs)
+ {
+ n = NiceNumGeneral(Num * 1000 / ElapsedMs, Buffer, kNicenumBytes);
+ }
+ else
+ {
+ Buffer[n++] = '0';
+ Buffer[n++] = 'B';
+ }
Buffer[n++] = '/';
Buffer[n++] = 's';
diff --git a/zencore/testutils.cpp b/zencore/testutils.cpp
index 78957cbcf..dbc3ab5af 100644
--- a/zencore/testutils.cpp
+++ b/zencore/testutils.cpp
@@ -26,6 +26,13 @@ ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporar
{
}
+ScopedTemporaryDirectory::ScopedTemporaryDirectory(std::filesystem::path Directory) : m_RootPath(Directory)
+{
+ std::error_code Ec;
+ std::filesystem::remove_all(Directory, Ec);
+ std::filesystem::create_directories(Directory);
+}
+
ScopedTemporaryDirectory::~ScopedTemporaryDirectory()
{
std::error_code Ec;
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 5c4983472..f14bc7896 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -1403,7 +1403,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
{
return SharedBuffer(std::move(Data));
}
- else if (Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath.native().c_str()))
+ else if (Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath))
{
return SharedBuffer(std::move(Data));
}
diff --git a/zenserver/vfs.cpp b/zenserver/vfs.cpp
deleted file mode 100644
index fcc9a71f8..000000000
--- a/zenserver/vfs.cpp
+++ /dev/null
@@ -1,900 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "vfs.h"
-
-#if ZEN_WITH_VFS
-# include <zencore/except.h>
-# include <zencore/filesystem.h>
-# include <zencore/stream.h>
-# include <zencore/windows.h>
-# include <zencore/logging.h>
-# include <zenstore/CAS.h>
-
-# include <map>
-
-# include <atlfile.h>
-# include <projectedfslib.h>
-
-# pragma comment(lib, "projectedfslib.lib")
-
-namespace zen {
-
-//////////////////////////////////////////////////////////////////////////
-
-struct ProjFsCliOptions
-{
- bool IsDebug = false;
- bool IsClean = false;
- std::string CasSpec;
- std::string ManifestSpec;
- std::string MountPoint;
-};
-
-struct GuidHasher
-{
- size_t operator()(const GUID& Guid) const
- {
- static_assert(sizeof(GUID) == (sizeof(size_t) * 2));
-
- const size_t* Ptr = reinterpret_cast<const size_t*>(&Guid);
-
- return Ptr[0] ^ Ptr[1];
- }
-};
-
-class ProjfsNamespace
-{
-public:
- HRESULT Initialize(const char* SnapshotSpec, const char* CasSpec)
- {
- std::filesystem::path ManifestSpec = zen::ManifestSpecToPath(SnapshotSpec);
-
- CAtlFile ManifestFile;
- HRESULT hRes = ManifestFile.Create(ManifestSpec.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
- if (FAILED(hRes))
- {
- ZEN_ERROR("MANIFEST NOT FOUND!"); // TODO: add context
-
- return hRes;
- }
-
- ULONGLONG FileLength = 0;
- ManifestFile.GetSize(FileLength);
-
- std::vector<uint8_t> Data;
- Data.resize(FileLength);
-
- ManifestFile.Read(Data.data(), (DWORD)Data.size());
-
- zen::MemoryInStream MemoryStream(Data.data(), Data.size());
-
- ReadManifest(/* out */ m_Manifest, MemoryStream);
-
- uint64_t TotalBytes = 0;
- uint64_t TotalFiles = 0;
-
- m_Manifest.Root.VisitFiles([&](const zen::LeafNode& Node) {
- TotalBytes += Node.FileSize;
- TotalFiles++;
- });
-
- m_FileByteCount = TotalBytes;
- m_FileCount = TotalFiles;
-
- // CAS root
-
- zen::CasStoreConfiguration Config;
- Config.RootDirectory = CasSpec;
- m_CasStore->Initialize(Config);
-
- return S_OK;
- }
-
- struct LookupResult
- {
- const zen::TreeNode* TreeNode = nullptr;
- const zen::LeafNode* LeafNode = nullptr;
- };
-
- bool IsOnCasDrive(const char* Path)
- {
- ZEN_UNUSED(Path);
-
- // TODO: programmatically determine of CAS and workspace path is on same drive!
- return true;
- }
-
- LookupResult LookupNode(const std::wstring& Name) const
- {
- if (Name.empty())
- return {nullptr};
-
- zen::ExtendableWideStringBuilder<MAX_PATH> LocalName;
- LocalName.Append(Name.c_str());
-
- // Split components
-
- const wchar_t* PathComponents[MAX_PATH / 2];
- size_t PathComponentCount = 0;
-
- const size_t Length = Name.length();
-
- wchar_t* Base = LocalName.Data();
- wchar_t* itStart = Base;
-
- for (int i = 0; i < Length; ++i)
- {
- if (Base[i] == '\\')
- {
- // Component separator
-
- Base[i] = L'\0';
-
- PathComponents[PathComponentCount++] = itStart;
-
- itStart = Base + i + 1;
- }
- }
-
- // Push final component
- if (Name.back() != L'\\')
- PathComponents[PathComponentCount++] = itStart;
-
- const zen::TreeNode* Node = &m_Manifest.Root;
-
- if (PathComponentCount == 1)
- {
- if (PrjFileNameCompare(L"root", Name.c_str()) == 0)
- return {Node};
- else
- return {nullptr};
- }
-
- for (size_t i = 1; i < PathComponentCount; ++i)
- {
- const auto& part = PathComponents[i];
-
- const zen::TreeNode* NextNode = nullptr;
-
- for (const zen::TreeNode& ChildNode : Node->Children)
- {
- if (PrjFileNameCompare(part, ChildNode.Name.c_str()) == 0)
- {
- NextNode = &ChildNode;
- break;
- }
- }
-
- if (NextNode)
- {
- Node = NextNode;
-
- continue;
- }
-
- if (i == PathComponentCount - 1)
- {
- for (const zen::LeafNode& Leaf : Node->Leaves)
- {
- if (PrjFileNameCompare(part, Leaf.Name.c_str()) == 0)
- return {nullptr, &Leaf};
- }
- }
-
- return {nullptr};
- }
-
- return {Node};
- }
-
- const zen::SnapshotManifest& Manifest() const { return m_Manifest; }
- zen::CasStore& CasStore() { return *m_CasStore; }
-
- uint64_t FileCount() const { return m_FileCount; }
- uint64_t FileByteCount() const { return m_FileByteCount; }
-
-private:
- zen::SnapshotManifest m_Manifest;
- std::unique_ptr<zen::CasStore> m_CasStore;
-
- size_t m_FileCount = 0;
- size_t m_FileByteCount = 0;
-};
-
-/** Projected File System Provider
- */
-
-class ProjfsProvider
-{
-public:
- HRESULT ReadManifest(const char* ManifestSpec, const char* CasSpec);
- HRESULT Initialize(std::filesystem::path RootPath, bool Clean);
- void Cleanup();
-
- struct Callbacks;
-
-private:
- static void DebugPrint(const char* Format, ...);
-
- HRESULT StartDirEnum(const PRJ_CALLBACK_DATA* CallbackData, LPCGUID EnumerationId);
- HRESULT EndDirEnum(const PRJ_CALLBACK_DATA* CallbackData, LPCGUID EnumerationId);
- HRESULT GetDirEnum(const PRJ_CALLBACK_DATA* CallbackData,
- LPCGUID EnumerationId,
- LPCWSTR SearchExpression,
- PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle);
- HRESULT GetPlaceholderInformation(const PRJ_CALLBACK_DATA* CallbackData);
- HRESULT GetFileStream(const PRJ_CALLBACK_DATA* CallbackData, UINT64 ByteOffset, UINT32 Length);
- HRESULT QueryFileName(const PRJ_CALLBACK_DATA* CallbackData);
- HRESULT NotifyOperation(const PRJ_CALLBACK_DATA* CallbackData,
- BOOLEAN IsDirectory,
- PRJ_NOTIFICATION NotificationType,
- LPCWSTR DestinationFileName,
- PRJ_NOTIFICATION_PARAMETERS* OperationParameters);
- void CancelCommand(const PRJ_CALLBACK_DATA* CallbackData);
-
- class DirectoryEnumeration;
-
- zen::RwLock m_Lock;
- std::unordered_map<GUID, std::unique_ptr<DirectoryEnumeration>, GuidHasher> m_DirectoryEnumerators;
- ProjfsNamespace m_Namespace;
- PRJ_NAMESPACE_VIRTUALIZATION_CONTEXT m_PrjContext = nullptr;
- bool m_GenerateFullFiles = false;
-};
-
-class ProjfsProvider::DirectoryEnumeration
-{
-public:
- DirectoryEnumeration(ProjfsProvider* Outer, LPCGUID EnumerationGuid, const wchar_t* RelativePath)
- : m_Outer(Outer)
- , m_EnumerationId(*EnumerationGuid)
- , m_Path(RelativePath)
- {
- ResetScan();
- }
-
- ~DirectoryEnumeration() {}
-
- void ResetScan()
- {
- // Restart enumeration from beginning
-
- m_InfoIterator = m_Infos.end();
-
- const ProjfsNamespace::LookupResult Lookup = m_Outer->m_Namespace.LookupNode(m_Path);
-
- if (Lookup.TreeNode == nullptr && Lookup.LeafNode == nullptr)
- return;
-
- if (Lookup.TreeNode)
- {
- const zen::TreeNode* RootNode = Lookup.TreeNode;
-
- // Populate info array
-
- FILETIME FileTime;
- GetSystemTimeAsFileTime(&FileTime);
-
- for (const zen::TreeNode& ChildNode : RootNode->Children)
- {
- PRJ_FILE_BASIC_INFO Fbi{0};
-
- Fbi.IsDirectory = TRUE;
- Fbi.FileSize = 0;
- Fbi.CreationTime = Fbi.LastAccessTime = Fbi.LastWriteTime = Fbi.ChangeTime = *((LARGE_INTEGER*)&FileTime);
- Fbi.FileAttributes = FILE_ATTRIBUTE_DIRECTORY;
-
- m_Infos.insert({ChildNode.Name, Fbi});
- }
-
- for (const zen::LeafNode& Leaf : RootNode->Leaves)
- {
- PRJ_FILE_BASIC_INFO Fbi{0};
-
- Fbi.IsDirectory = FALSE;
- Fbi.FileSize = Leaf.FileSize;
- Fbi.FileAttributes = FILE_ATTRIBUTE_NORMAL;
- Fbi.CreationTime = Fbi.LastAccessTime = Fbi.LastWriteTime = Fbi.ChangeTime =
- *reinterpret_cast<const LARGE_INTEGER*>(&Leaf.FileModifiedTime);
-
- m_Infos.insert({Leaf.Name, Fbi});
- }
- }
-
- m_InfoIterator = m_Infos.begin();
- }
-
- HRESULT HandleRequest(_In_ const PRJ_CALLBACK_DATA* CallbackData,
- _In_opt_z_ LPCWSTR SearchExpression,
- _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle)
- {
- int EnumLimit = INT_MAX;
-
- DebugPrint("ENUM '%S' -> pattern %S\n", CallbackData->FilePathName, SearchExpression);
-
- HRESULT hRes = S_OK;
-
- if (CallbackData->Flags & PRJ_CB_DATA_FLAG_ENUM_RESTART_SCAN)
- ResetScan();
-
- if (m_InfoIterator == m_Infos.end())
- return S_OK;
-
- if (CallbackData->Flags & PRJ_CB_DATA_FLAG_ENUM_RETURN_SINGLE_ENTRY)
- EnumLimit = 1;
-
- if (!m_Predicate)
- {
- if (SearchExpression)
- {
- bool isWild = PrjDoesNameContainWildCards(SearchExpression);
-
- if (isWild)
- {
- if (SearchExpression[0] == L'*' && SearchExpression[1] == L'\0')
- {
- // Trivial accept -- no need to change predicate from the default
- }
- else
- {
- m_SearchExpression = SearchExpression;
-
- m_Predicate = [this](LPCWSTR name) { return PrjFileNameMatch(name, m_SearchExpression.c_str()); };
- }
- }
- else
- {
- if (SearchExpression[0])
- {
- // Look for specific name match (does this ever happen?)
-
- m_SearchExpression = SearchExpression;
-
- m_Predicate = [this](LPCWSTR name) { return PrjFileNameCompare(name, m_SearchExpression.c_str()) == 0; };
- }
- }
- }
- }
-
- if (!m_Predicate)
- m_Predicate = [](LPCWSTR) { return true; };
-
- while (EnumLimit && m_InfoIterator != m_Infos.end())
- {
- auto& ThisNode = *m_InfoIterator;
-
- auto& Name = ThisNode.first;
- auto& Info = ThisNode.second;
-
- if (m_Predicate(Name.c_str()))
- {
- hRes = PrjFillDirEntryBuffer(Name.c_str(), &Info, DirEntryBufferHandle);
-
- if (hRes == HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER))
- return S_OK;
-
- if (FAILED(hRes))
- break;
-
- --EnumLimit;
- }
-
- ++m_InfoIterator;
- }
-
- return hRes;
- }
-
-private:
- ProjfsProvider* m_Outer = nullptr;
- const std::wstring m_Path;
- const GUID m_EnumerationId;
-
- // We need to maintain an ordered list of directory items since the
- // ProjFS enumeration code gets confused otherwise and ends up producing
- // multiple entries for the same file if there's a 'hydrated' version
- // present.
-
- struct FilenameLess
- {
- bool operator()(const std::wstring& Lhs, const std::wstring& Rhs) const { return PrjFileNameCompare(Lhs.c_str(), Rhs.c_str()) < 0; }
- };
-
- typedef std::map<std::wstring, PRJ_FILE_BASIC_INFO, FilenameLess> FileInfoMap_t;
-
- FileInfoMap_t m_Infos;
- FileInfoMap_t::iterator m_InfoIterator;
-
- std::wstring m_SearchExpression;
- std::function<bool(LPCWSTR name)> m_Predicate;
-};
-
-//////////////////////////////////////////////////////////////////////////
-// Callback forwarding functions
-//
-
-struct ProjfsProvider::Callbacks
-{
- static HRESULT CALLBACK StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
- {
- return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->StartDirEnum(CallbackData, EnumerationId);
- }
-
- static HRESULT CALLBACK EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ LPCGUID EnumerationId)
- {
- return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->EndDirEnum(CallbackData, EnumerationId);
- }
-
- static HRESULT CALLBACK GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData,
- _In_ LPCGUID EnumerationId,
- _In_opt_z_ LPCWSTR SearchExpression,
- _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle)
- {
- return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)
- ->GetDirEnum(CallbackData, EnumerationId, SearchExpression, DirEntryBufferHandle);
- }
-
- static HRESULT CALLBACK GetPlaceholderInformation(_In_ const PRJ_CALLBACK_DATA* CallbackData)
- {
- return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->GetPlaceholderInformation(CallbackData);
- }
-
- static HRESULT CALLBACK GetFileStream(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length)
- {
- return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->GetFileStream(CallbackData, ByteOffset, Length);
- }
-
- static HRESULT CALLBACK QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData)
- {
- return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->QueryFileName(CallbackData);
- }
-
- static HRESULT CALLBACK NotifyOperation(_In_ const PRJ_CALLBACK_DATA* CallbackData,
- _In_ BOOLEAN IsDirectory,
- _In_ PRJ_NOTIFICATION NotificationType,
- _In_opt_ LPCWSTR DestinationFileName,
- _Inout_ PRJ_NOTIFICATION_PARAMETERS* OperationParameters)
- {
- return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)
- ->NotifyOperation(CallbackData, IsDirectory, NotificationType, DestinationFileName, OperationParameters);
- }
-
- static VOID CALLBACK CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData)
- {
- return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->CancelCommand(CallbackData);
- }
-};
-
-// {6EEB94E4-3EF3-4C1C-AF15-D7FF64C19A4F}
-static const GUID ProviderGuid = {0x6eeb94e4, 0x3ef3, 0x4c1c, {0xaf, 0x15, 0xd7, 0xff, 0x64, 0xc1, 0x9a, 0x4f}};
-
-void
-ProjfsProvider::DebugPrint(const char* FmtString, ...)
-{
- va_list vl;
- va_start(vl, FmtString);
-
-# if 0
- vprintf(FmtString, vl);
-# endif
-
- va_end(vl);
-}
-
-HRESULT
-ProjfsProvider::Initialize(std::filesystem::path RootPath, bool Clean)
-{
- PRJ_PLACEHOLDER_VERSION_INFO Pvi = {};
- Pvi.ContentID[0] = 1;
-
- if (Clean && std::filesystem::exists(RootPath))
- {
- printf("Cleaning '%S'...", RootPath.c_str());
-
- bool success = zen::DeleteDirectories(RootPath);
-
- if (!success)
- {
- printf(" retrying...");
-
- success = zen::DeleteDirectories(RootPath);
-
- // Failed?
- }
-
- printf(" done!\n");
- }
-
- bool RootDirectoryCreated = false;
-
-retry:
- if (!std::filesystem::exists(RootPath))
- {
- zen::CreateDirectories(RootPath);
- }
-
- {
- HRESULT hRes = PrjMarkDirectoryAsPlaceholder(RootPath.c_str(), nullptr, &Pvi, &ProviderGuid);
-
- if (FAILED(hRes))
- {
- if (hRes == HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND) && !RootDirectoryCreated)
- {
- printf("Creating '%S'...", RootPath.c_str());
-
- std::filesystem::create_directories(RootPath.c_str());
-
- RootDirectoryCreated = true;
-
- printf("done!\n");
-
- goto retry;
- }
- else if (hRes == HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND))
- {
- ThrowSystemException(hRes, "Failed to initialize root placeholder");
- }
-
- // Ignore error, problems will be reported below anyway
- }
- }
-
- // Callbacks
-
- PRJ_CALLBACKS cbs = {};
-
- cbs.StartDirectoryEnumerationCallback = Callbacks::StartDirEnum;
- cbs.EndDirectoryEnumerationCallback = Callbacks::EndDirEnum;
- cbs.GetDirectoryEnumerationCallback = Callbacks::GetDirEnum;
- cbs.GetPlaceholderInfoCallback = Callbacks::GetPlaceholderInformation;
- cbs.GetFileDataCallback = Callbacks::GetFileStream;
- cbs.QueryFileNameCallback = Callbacks::QueryFileName;
- cbs.NotificationCallback = Callbacks::NotifyOperation;
- cbs.CancelCommandCallback = Callbacks::CancelCommand;
-
- // Parameters
-
- const PRJ_NOTIFY_TYPES dwNotifications = PRJ_NOTIFY_FILE_OPENED | PRJ_NOTIFY_NEW_FILE_CREATED | PRJ_NOTIFY_FILE_OVERWRITTEN |
- PRJ_NOTIFY_PRE_DELETE | PRJ_NOTIFY_PRE_RENAME | PRJ_NOTIFY_PRE_SET_HARDLINK |
- PRJ_NOTIFY_FILE_RENAMED | PRJ_NOTIFY_HARDLINK_CREATED |
- PRJ_NOTIFY_FILE_HANDLE_CLOSED_NO_MODIFICATION | PRJ_NOTIFY_FILE_HANDLE_CLOSED_FILE_MODIFIED |
- PRJ_NOTIFY_FILE_HANDLE_CLOSED_FILE_DELETED | PRJ_NOTIFY_FILE_PRE_CONVERT_TO_FULL;
-
- PRJ_NOTIFICATION_MAPPING Mappings[] = {{dwNotifications, L"root"}};
-
- PRJ_STARTVIRTUALIZING_OPTIONS SvOptions = {};
-
- SvOptions.Flags = PRJ_FLAG_NONE;
- SvOptions.PoolThreadCount = 8;
- SvOptions.ConcurrentThreadCount = 8;
- SvOptions.NotificationMappings = Mappings;
- SvOptions.NotificationMappingsCount = 1;
-
- HRESULT hRes = PrjStartVirtualizing(RootPath.c_str(), &cbs, this, &SvOptions, &m_PrjContext);
-
- if (SUCCEEDED(hRes))
- {
- // Create dummy 'root' directory for now until I figure out how to
- // invalidate entire trees (ProjFS won't allow invalidation of the
- // entire provider tree).
-
- PRJ_PLACEHOLDER_INFO pli{};
- pli.FileBasicInfo.IsDirectory = TRUE;
- pli.FileBasicInfo.FileAttributes = FILE_ATTRIBUTE_DIRECTORY;
- pli.VersionInfo = Pvi;
-
- hRes = PrjWritePlaceholderInfo(m_PrjContext, L"root", &pli, sizeof pli);
- }
-
- if (SUCCEEDED(hRes))
- {
- ZEN_INFO("Successfully mounted snapshot at '{}'!", WideToUtf8(RootPath.c_str()));
- }
- else
- {
- ZEN_INFO("Failed mounting snapshot at '{}'!", WideToUtf8(RootPath.c_str()));
- }
-
- return hRes;
-}
-
-void
-ProjfsProvider::Cleanup()
-{
- PrjStopVirtualizing(m_PrjContext);
-}
-
-HRESULT
-ProjfsProvider::ReadManifest(const char* ManifestSpec, const char* CasSpec)
-{
- printf("Initializing from manifest '%s'\n", ManifestSpec);
-
- m_Namespace.Initialize(ManifestSpec, CasSpec);
-
- return S_OK;
-}
-
-HRESULT
-ProjfsProvider::StartDirEnum(const PRJ_CALLBACK_DATA* CallbackData, LPCGUID EnumerationId)
-{
- zen::RwLock::ExclusiveLockScope _(m_Lock);
-
- m_DirectoryEnumerators[*EnumerationId] = std::make_unique<DirectoryEnumeration>(this, EnumerationId, CallbackData->FilePathName);
-
- return S_OK;
-}
-
-HRESULT
-ProjfsProvider::EndDirEnum(const PRJ_CALLBACK_DATA* CallbackData, LPCGUID EnumerationId)
-{
- ZEN_UNUSED(CallbackData);
- ZEN_UNUSED(EnumerationId);
-
- zen::RwLock::ExclusiveLockScope _(m_Lock);
-
- m_DirectoryEnumerators.erase(*EnumerationId);
-
- return S_OK;
-}
-
-HRESULT
-ProjfsProvider::GetDirEnum(const PRJ_CALLBACK_DATA* CallbackData,
- LPCGUID EnumerationId,
- LPCWSTR SearchExpression,
- PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle)
-{
- DirectoryEnumeration* directoryEnumerator;
-
- {
- zen::RwLock::SharedLockScope _(m_Lock);
-
- auto it = m_DirectoryEnumerators.find(*EnumerationId);
-
- if (it == m_DirectoryEnumerators.end())
- return E_FAIL; // No enumerator associated with specified GUID
-
- directoryEnumerator = (*it).second.get();
- }
-
- return directoryEnumerator->HandleRequest(CallbackData, SearchExpression, DirEntryBufferHandle);
-}
-
-HRESULT
-ProjfsProvider::GetPlaceholderInformation(const PRJ_CALLBACK_DATA* CallbackData)
-{
- ProjfsNamespace::LookupResult result = m_Namespace.LookupNode(CallbackData->FilePathName);
-
- if (auto Leaf = result.LeafNode)
- {
- PRJ_PLACEHOLDER_INFO PlaceholderInfo = {};
-
- LARGE_INTEGER FileTime;
- FileTime.QuadPart = Leaf->FileModifiedTime;
-
- PlaceholderInfo.FileBasicInfo.ChangeTime = FileTime;
- PlaceholderInfo.FileBasicInfo.CreationTime = FileTime;
- PlaceholderInfo.FileBasicInfo.LastAccessTime = FileTime;
- PlaceholderInfo.FileBasicInfo.LastWriteTime = FileTime;
- PlaceholderInfo.FileBasicInfo.FileSize = Leaf->FileSize;
- PlaceholderInfo.FileBasicInfo.IsDirectory = 0;
- PlaceholderInfo.FileBasicInfo.FileAttributes = FILE_ATTRIBUTE_NORMAL;
-
- HRESULT hRes = PrjWritePlaceholderInfo(m_PrjContext, CallbackData->FilePathName, &PlaceholderInfo, sizeof PlaceholderInfo);
-
- return hRes;
- }
-
- if (auto node = result.TreeNode)
- {
- PRJ_PLACEHOLDER_INFO PlaceholderInfo = {};
-
- FILETIME ft;
- GetSystemTimeAsFileTime(&ft);
-
- LARGE_INTEGER FileTime;
- FileTime.QuadPart = UINT64(ft.dwHighDateTime) << 32 | ft.dwLowDateTime;
-
- PlaceholderInfo.FileBasicInfo.ChangeTime = FileTime;
- PlaceholderInfo.FileBasicInfo.CreationTime = FileTime;
- PlaceholderInfo.FileBasicInfo.LastAccessTime = FileTime;
- PlaceholderInfo.FileBasicInfo.LastWriteTime = FileTime;
- PlaceholderInfo.FileBasicInfo.IsDirectory = TRUE;
- PlaceholderInfo.FileBasicInfo.FileAttributes = FILE_ATTRIBUTE_DIRECTORY;
-
- HRESULT hRes = PrjWritePlaceholderInfo(m_PrjContext, CallbackData->FilePathName, &PlaceholderInfo, sizeof PlaceholderInfo);
-
- return hRes;
- }
-
- return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
-}
-
-HRESULT
-ProjfsProvider::GetFileStream(const PRJ_CALLBACK_DATA* CallbackData, UINT64 ByteOffset, UINT32 Length)
-{
- ProjfsNamespace::LookupResult result = m_Namespace.LookupNode(CallbackData->FilePathName);
-
- if (const zen::LeafNode* leaf = result.LeafNode)
- {
- zen::CasStore& casStore = m_Namespace.CasStore();
-
- const zen::IoHash& ChunkHash = leaf->ChunkHash;
-
- zen::IoBuffer Chunk = casStore.FindChunk(ChunkHash);
-
- if (!Chunk)
- return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
-
- if (m_GenerateFullFiles)
- {
- DWORD chunkSize = (DWORD)Chunk.Size();
-
- zen::StringBuilder<66> b3string;
- DebugPrint("GET FILE STREAM: %s -> %d '%S'\n", ChunkHash.ToHexString(b3string).c_str(), chunkSize, CallbackData->FilePathName);
-
- // TODO: implement support for chunks > 4GB
- ZEN_ASSERT(chunkSize == Chunk.Size());
-
- HRESULT hRes = PrjWriteFileData(m_PrjContext, &CallbackData->DataStreamId, (PVOID)Chunk.Data(), 0, chunkSize);
-
- return hRes;
- }
- else
- {
- HRESULT hRes = PrjWriteFileData(m_PrjContext,
- &CallbackData->DataStreamId,
- (PVOID)(reinterpret_cast<const uint8_t*>(Chunk.Data()) + ByteOffset),
- ByteOffset,
- Length);
-
- return hRes;
- }
- }
-
- return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
-}
-
-HRESULT
-ProjfsProvider::QueryFileName(const PRJ_CALLBACK_DATA* CallbackData)
-{
- ProjfsNamespace::LookupResult result = m_Namespace.LookupNode(CallbackData->FilePathName);
-
- if (result.LeafNode || result.TreeNode)
- return S_OK;
-
- return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
-}
-
-HRESULT
-ProjfsProvider::NotifyOperation(const PRJ_CALLBACK_DATA* CallbackData,
- BOOLEAN IsDirectory,
- PRJ_NOTIFICATION NotificationType,
- LPCWSTR DestinationFileName,
- PRJ_NOTIFICATION_PARAMETERS* OperationParameters)
-{
- ZEN_UNUSED(DestinationFileName);
-
- switch (NotificationType)
- {
- case PRJ_NOTIFICATION_FILE_OPENED:
- {
- auto& pc = OperationParameters->PostCreate;
-
- DebugPrint("*** OPEN: %s %08x '%S'\n", IsDirectory ? "(DIR)" : "-FILE", pc.NotificationMask, CallbackData->FilePathName);
- }
- break;
-
- case PRJ_NOTIFICATION_NEW_FILE_CREATED:
- {
- auto& pc = OperationParameters->PostCreate;
-
- DebugPrint("*** NEW : %s %08x '%S'\n", IsDirectory ? "(DIR)" : "-FILE", pc.NotificationMask, CallbackData->FilePathName);
- }
- break;
-
- case PRJ_NOTIFICATION_FILE_OVERWRITTEN:
- {
- auto& pc = OperationParameters->PostCreate;
-
- DebugPrint("*** OVER: %s %08x '%S'\n", IsDirectory ? "(DIR)" : "-FILE", pc.NotificationMask, CallbackData->FilePathName);
- }
- break;
-
- case PRJ_NOTIFICATION_PRE_DELETE:
- {
- if (wcsstr(CallbackData->FilePathName, L"en-us"))
- DebugPrint("*** PRE DELETE '%S'\n", CallbackData->FilePathName);
-
- DebugPrint("*** PRE DELETE '%S'\n", CallbackData->FilePathName);
- }
- break;
-
- case PRJ_NOTIFICATION_PRE_RENAME:
- DebugPrint("*** PRE RENAME '%S'\n", CallbackData->FilePathName);
- break;
-
- case PRJ_NOTIFICATION_PRE_SET_HARDLINK:
- DebugPrint("*** PRE SET HARDLINK '%S'\n", CallbackData->FilePathName);
- break;
-
- case PRJ_NOTIFICATION_FILE_RENAMED:
- DebugPrint("*** FILE RENAMED '%S'\n", CallbackData->FilePathName);
- break;
-
- case PRJ_NOTIFICATION_HARDLINK_CREATED:
- DebugPrint("*** HARDLINK RENAMED '%S'\n", CallbackData->FilePathName);
- break;
-
- case PRJ_NOTIFICATION_FILE_HANDLE_CLOSED_NO_MODIFICATION:
- DebugPrint("*** FILE CLOSED NO CHANGE '%S'\n", CallbackData->FilePathName);
- break;
-
- case PRJ_NOTIFICATION_FILE_HANDLE_CLOSED_FILE_MODIFIED:
- {
- // const auto& handleClose = OperationParameters->FileDeletedOnHandleClose;
-
- DebugPrint("*** FILE CLOSED MODIFIED '%S'\n", CallbackData->FilePathName);
- }
- break;
-
- case PRJ_NOTIFICATION_FILE_HANDLE_CLOSED_FILE_DELETED:
- {
- // const auto& handleClose = OperationParameters->FileDeletedOnHandleClose;
-
- DebugPrint("*** FILE CLOSED DELETED '%S'\n", CallbackData->FilePathName);
- }
- break;
-
- case PRJ_NOTIFICATION_FILE_PRE_CONVERT_TO_FULL:
- DebugPrint("*** FILE PRE CONVERT FULL '%S'\n", CallbackData->FilePathName);
- break;
- }
-
- return S_OK;
-}
-
-void
-ProjfsProvider::CancelCommand(const PRJ_CALLBACK_DATA* CallbackData)
-{
- ZEN_UNUSED(CallbackData);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-struct Vfs::VfsImpl
-{
- void Initialize() { m_PrjProvider.Initialize("E:\\VFS_Test", /* clean */ true); }
- void Start() {}
- void Stop() {}
-
-private:
- ProjfsProvider m_PrjProvider;
-};
-
-//////////////////////////////////////////////////////////////////////////
-
-Vfs::Vfs() : m_Impl(new VfsImpl)
-{
-}
-
-Vfs::~Vfs()
-{
-}
-
-void
-Vfs::Initialize()
-{
- m_Impl->Initialize();
-}
-
-void
-Vfs::Start()
-{
-}
-
-void
-Vfs::Stop()
-{
-}
-
-} // namespace zen
-#endif \ No newline at end of file
diff --git a/zenserver/vfs.h b/zenserver/vfs.h
deleted file mode 100644
index 0d2ca6062..000000000
--- a/zenserver/vfs.h
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#ifndef ZEN_WITH_VFS
-# define ZEN_WITH_VFS 0
-#endif
-
-#if ZEN_WITH_VFS
-# include <memory>
-
-namespace zen {
-
-/**
- * Virtual File System serving
- */
-
-class Vfs
-{
-public:
- Vfs();
- ~Vfs();
-
- void Initialize();
-
- void Start();
- void Stop();
-
-private:
- struct VfsImpl;
-
- std::unique_ptr<VfsImpl> m_Impl;
-};
-
-} // namespace zen
-#endif
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index 7fad477a1..b670582e7 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -83,6 +83,7 @@
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<UACExecutionLevel>RequireAdministrator</UACExecutionLevel>
+ <DelayLoadDLLs>projectedfslib.dll;shell32.dll</DelayLoadDLLs>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@@ -99,7 +100,7 @@
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<UACExecutionLevel>RequireAdministrator</UACExecutionLevel>
- <DelayLoadDLLs>projectedfslib.dll</DelayLoadDLLs>
+ <DelayLoadDLLs>projectedfslib.dll;shell32.dll</DelayLoadDLLs>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
@@ -127,7 +128,6 @@
<ClInclude Include="targetver.h" />
<ClInclude Include="upstream\upstreamcache.h" />
<ClInclude Include="upstream\zen.h" />
- <ClInclude Include="vfs.h" />
<ClInclude Include="windows\service.h" />
</ItemGroup>
<ItemGroup>
@@ -151,7 +151,6 @@
<ClCompile Include="experimental\usnjournal.cpp" />
<ClCompile Include="upstream\upstreamcache.cpp" />
<ClCompile Include="upstream\zen.cpp" />
- <ClCompile Include="vfs.cpp" />
<ClCompile Include="windows\service.cpp" />
<ClCompile Include="zenserver.cpp" />
</ItemGroup>
diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters
index 04e639a33..b87fa0016 100644
--- a/zenserver/zenserver.vcxproj.filters
+++ b/zenserver/zenserver.vcxproj.filters
@@ -4,7 +4,6 @@
<ClInclude Include="targetver.h" />
<ClInclude Include="projectstore.h" />
<ClInclude Include="casstore.h" />
- <ClInclude Include="vfs.h" />
<ClInclude Include="testing\launch.h" />
<ClInclude Include="cache\cacheagent.h">
<Filter>cache</Filter>
@@ -50,7 +49,6 @@
<ClCompile Include="zenserver.cpp" />
<ClCompile Include="projectstore.cpp" />
<ClCompile Include="casstore.cpp" />
- <ClCompile Include="vfs.cpp" />
<ClCompile Include="cache\cacheagent.cpp">
<Filter>cache</Filter>
</ClCompile>
diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp
index f41f04101..cfc77e2a5 100644
--- a/zenstore/basicfile.cpp
+++ b/zenstore/basicfile.cpp
@@ -143,6 +143,12 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<voi
}
}
+void
+BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec)
+{
+ Write(Data.GetData(), Data.GetSize(), FileOffset, Ec);
+}
+
void
BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec)
{
@@ -177,6 +183,12 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro
}
void
+BasicFile::Write(MemoryView Data, uint64_t FileOffset)
+{
+ Write(Data.GetData(), Data.GetSize(), FileOffset);
+}
+
+void
BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
{
std::error_code Ec;
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index ee641b80a..f69ed6bdb 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -9,12 +9,15 @@
#include <zencore/memory.h>
#include <zencore/scopeguard.h>
#include <zencore/string.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
#include <zencore/thread.h>
#include <zencore/uid.h>
#include <zenstore/basicfile.h>
#include <gsl/gsl-lite.hpp>
+#include <barrier>
#include <filesystem>
#include <functional>
#include <unordered_map>
@@ -86,15 +89,19 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
{
ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);
+ const HANDLE ChunkFileHandle = FileRef.FileHandle;
+
auto DeletePayloadFileOnClose = [&] {
// This will cause the file to be deleted when the last handle to it is closed
FILE_DISPOSITION_INFO Fdi{};
Fdi.DeleteFile = TRUE;
- BOOL Success = SetFileInformationByHandle(FileRef.FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);
+ BOOL Success = SetFileInformationByHandle(ChunkFileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);
if (!Success)
{
- ZEN_WARN("Failed to flag temporary payload file for deletion: '{}'", PathFromHandle(FileRef.FileHandle));
+ ZEN_WARN("Failed to flag temporary payload file '{}' for deletion: '{}'",
+ PathFromHandle(ChunkFileHandle),
+ GetLastErrorAsString());
}
};
@@ -118,6 +125,28 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
return CasStore::InsertResult{.New = false};
}
+ else
+ {
+ if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND))
+ {
+ // Shard directory does not exist
+ }
+ else if (hRes == HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND))
+ {
+ // Shard directory exists, but not the file
+ }
+ else if (hRes == HRESULT_FROM_WIN32(ERROR_SHARING_VIOLATION))
+ {
+ // Sharing violation, likely because we are trying to open a file
+ // which has been renamed on another thread, and the file handle
+ // used to rename it is still open. We handle this case below
+ // instead of here
+ }
+ else
+ {
+ ZEN_INFO("Unexpected error opening file '{}': {}", WideToUtf8(Name.ShardedPath), hRes);
+ }
+ }
}
std::filesystem::path FullPath(Name.ShardedPath.c_str());
@@ -138,7 +167,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
// Try to move file into place
- BOOL Success = SetFileInformationByHandle(FileRef.FileHandle, FileRenameInfo, RenameInfo, BufferSize);
+ BOOL Success = SetFileInformationByHandle(ChunkFileHandle, FileRenameInfo, RenameInfo, BufferSize);
if (!Success)
{
@@ -150,7 +179,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
auto InternalCreateDirectoryHandle = [&] {
return DirHandle.Create(FilePath.c_str(),
GENERIC_READ | GENERIC_WRITE,
- FILE_SHARE_READ | FILE_SHARE_WRITE,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS);
};
@@ -163,6 +192,9 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
if (FAILED(hRes))
{
+ // TODO: we can handle directory creation more intelligently and efficiently than
+ // this currently does
+
CreateDirectories(FilePath.c_str());
hRes = InternalCreateDirectoryHandle();
@@ -173,9 +205,9 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
ThrowSystemException(hRes, "Failed to open shard directory '{}'"_format(FilePath));
}
- // Retry
+ // Retry rename/move
- Success = SetFileInformationByHandle(FileRef.FileHandle, FileRenameInfo, RenameInfo, BufferSize);
+ Success = SetFileInformationByHandle(ChunkFileHandle, FileRenameInfo, RenameInfo, BufferSize);
}
if (Success)
@@ -183,8 +215,17 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
return CasStore::InsertResult{.New = true};
}
+ const DWORD LastError = GetLastError();
+
+ if ((LastError == ERROR_FILE_EXISTS) || (LastError == ERROR_ALREADY_EXISTS))
+ {
+ DeletePayloadFileOnClose();
+
+ return CasStore::InsertResult{.New = false};
+ }
+
ZEN_WARN("rename of CAS payload file failed ('{}'), falling back to regular write for insert of {}",
- GetLastErrorAsString(),
+ GetSystemErrorAsString(LastError),
ChunkHash);
DeletePayloadFileOnClose();
@@ -445,4 +486,94 @@ FileCasStrategy::GarbageCollect(GcContext& GcCtx)
ZEN_UNUSED(GcCtx);
}
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("cas.file.move")
+{
+ using namespace fmt::literals;
+
+ ScopedTemporaryDirectory TempDir{"d:\\filecas_testdir"};
+
+ CasStoreConfiguration CasConfig;
+ CasConfig.RootDirectory = TempDir.Path() / "cas";
+
+ FileCasStrategy FileCas(CasConfig);
+
+ {
+ std::filesystem::path Payload1Path{TempDir.Path() / "payload_1"};
+
+ IoBuffer ZeroBytes{1024 * 1024};
+ IoHash ZeroHash = IoHash::HashBuffer(ZeroBytes);
+
+ BasicFile PayloadFile;
+ PayloadFile.Open(Payload1Path, true);
+ PayloadFile.Write(ZeroBytes, 0);
+ PayloadFile.Close();
+
+ IoBuffer Payload1 = IoBufferBuilder::MakeFromTemporaryFile(Payload1Path);
+
+ CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, ZeroHash);
+ CHECK_EQ(Result.New, true);
+ }
+
+# if 0
+ SUBCASE("stresstest")
+ {
+ std::vector<IoHash> PayloadHashes;
+
+ const int kWorkers = 64;
+ const int kItemCount = 128;
+
+ for (int w = 0; w < kWorkers; ++w)
+ {
+ for (int i = 0; i < kItemCount; ++i)
+ {
+ IoBuffer Payload{1024};
+ *reinterpret_cast<int*>(Payload.MutableData()) = i;
+ PayloadHashes.push_back(IoHash::HashBuffer(Payload));
+
+ std::filesystem::path PayloadPath{TempDir.Path() / "payload_{}_{}"_format(w, i)};
+ WriteFile(PayloadPath, Payload);
+ }
+ }
+
+ std::barrier Sync{kWorkers};
+
+ auto PopulateAll = [&](int w) {
+ std::vector<IoBuffer> Buffers;
+
+ for (int i = 0; i < kItemCount; ++i)
+ {
+ std::filesystem::path PayloadPath{TempDir.Path() / "payload_{}_{}"_format(w, i)};
+ IoBuffer Payload = IoBufferBuilder::MakeFromTemporaryFile(PayloadPath);
+ Buffers.push_back(Payload);
+ Sync.arrive_and_wait();
+ CasStore::InsertResult Result = FileCas.InsertChunk(Payload, PayloadHashes[i]);
+ }
+ };
+
+ std::vector<std::jthread> Threads;
+
+ for (int i = 0; i < kWorkers; ++i)
+ {
+ Threads.push_back(std::jthread(PopulateAll, i));
+ }
+
+ for (std::jthread& Thread : Threads)
+ {
+ Thread.join();
+ }
+ }
+# endif
+}
+
+#endif
+
+void
+filecas_forcelink()
+{
+}
+
} // namespace zen
diff --git a/zenstore/filecas.h b/zenstore/filecas.h
index db21502c6..14314ce52 100644
--- a/zenstore/filecas.h
+++ b/zenstore/filecas.h
@@ -57,4 +57,6 @@ private:
};
};
+void filecas_forcelink();
+
} // namespace zen
diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h
index d0698df7f..86e7e78d9 100644
--- a/zenstore/include/zenstore/CAS.h
+++ b/zenstore/include/zenstore/CAS.h
@@ -21,7 +21,7 @@ namespace zen {
struct CasStoreConfiguration
{
- // Root directory for CAS store -- if not specified a default folder will be assigned in 'Documents\zen'
+ // Root directory for CAS store
std::filesystem::path RootDirectory;
// Threshold below which values are considered 'tiny' and managed using the 'tiny values' strategy
diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h
index fad4a33e1..7ae35dea6 100644
--- a/zenstore/include/zenstore/basicfile.h
+++ b/zenstore/include/zenstore/basicfile.h
@@ -36,6 +36,8 @@ public:
void Read(void* Data, uint64_t Size, uint64_t FileOffset);
void StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+ void Write(MemoryView Data, uint64_t FileOffset);
+ void Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec);
void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
void Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
void Flush();
diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp
index cd16e5634..d852fa64b 100644
--- a/zenstore/zenstore.cpp
+++ b/zenstore/zenstore.cpp
@@ -4,6 +4,7 @@
#include <zenstore/CAS.h>
#include <zenstore/basicfile.h>
+#include "filecas.h"
namespace zen {
@@ -12,6 +13,7 @@ zenstore_forcelinktests()
{
basicfile_forcelink();
CAS_forcelink();
+ filecas_forcelink();
}
} // namespace zen