diff options
| author | Stefan Boberg <[email protected]> | 2023-09-20 15:22:03 +0200 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-09-20 15:22:03 +0200 |
| commit | 14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f (patch) | |
| tree | bf24ac15759385cea339f7e1cf5380f984f5699a /src | |
| parent | changelog version bump (diff) | |
| download | zen-14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f.tar.xz zen-14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f.zip | |
VFS implementation for local storage service (#396)
currently, only Windows (using Projected File System) is supported
Diffstat (limited to 'src')
31 files changed, 2553 insertions, 27 deletions
diff --git a/src/zen/cmds/copy.h b/src/zen/cmds/copy.h index 5527ae9b8..549114160 100644 --- a/src/zen/cmds/copy.h +++ b/src/zen/cmds/copy.h @@ -18,11 +18,10 @@ public: virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; private: - cxxopts::Options m_Options{"copy", "Copy files"}; - std::vector<std::string> m_Positional; - std::string m_CopySource; - std::string m_CopyTarget; - bool m_NoClone = false; + cxxopts::Options m_Options{"copy", "Copy files"}; + std::string m_CopySource; + std::string m_CopyTarget; + bool m_NoClone = false; }; } // namespace zen diff --git a/src/zen/cmds/up.h b/src/zen/cmds/up.h index 5b5c6a3f8..510cc865e 100644 --- a/src/zen/cmds/up.h +++ b/src/zen/cmds/up.h @@ -17,8 +17,8 @@ public: private: cxxopts::Options m_Options{"up", "Bring up zen service"}; - uint16_t m_Port; - int m_OwnerPid; + uint16_t m_Port = 0; + int m_OwnerPid = 0; std::string m_ConfigFile; }; diff --git a/src/zen/cmds/vfs_cmd.cpp b/src/zen/cmds/vfs_cmd.cpp new file mode 100644 index 000000000..4eb9b3195 --- /dev/null +++ b/src/zen/cmds/vfs_cmd.cpp @@ -0,0 +1,108 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "vfs_cmd.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/string.h> +#include <zencore/uid.h> +#include <zenhttp/formatters.h> +#include <zenhttp/httpclient.h> +#include <zenutil/zenserverprocess.h> + +namespace zen { + +using namespace std::literals; + +VfsCommand::VfsCommand() +{ + m_Options.add_option("", "", "verb", "VFS management verb (mount, unmount)", cxxopts::value(m_Verb), "<verb>"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<url>"); + m_Options.add_option("", "", "vfs-path", "Specify VFS mount point path", cxxopts::value(m_MountPath), "<path>"); + + m_Options.parse_positional({"verb", "vfs-path"}); +} + +VfsCommand::~VfsCommand() +{ +} + +int +VfsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + + if (!ZenCmdBase::ParseOptions(argc, argv)) + { + return 0; + } + + // Validate arguments + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + throw OptionParseException("unable to resolve server specification"); + + HttpClient Http(m_HostName); + + if (m_Verb == "mount"sv) + { + if (m_MountPath.empty()) + throw OptionParseException("No source specified"); + + CbObjectWriter Cbo; + Cbo << "method" + << "mount"; + + Cbo.BeginObject("params"); + Cbo << "path" << m_MountPath; + Cbo.EndObject(); + + if (HttpClient::Response Result = Http.Post("/vfs"sv, Cbo.Save())) + { + } + else + { + Result.ThrowError("VFS mount request failed"sv); + + return 1; + } + } + else if (m_Verb == "unmount"sv) + { + CbObjectWriter Cbo; + Cbo << "method" + << "unmount"; + + if (HttpClient::Response Result = Http.Post("/vfs"sv, Cbo.Save())) + { + } + else + { + Result.ThrowError("VFS unmount request failed"sv); + + return 1; + } + } + else if (m_Verb == "info"sv) + { + if (HttpClient::Response Result = Http.Get(fmt::format("/vfs/info"))) + { + ExtendableStringBuilder<256> Json; + Result.AsObject().ToJson(Json); + ZEN_CONSOLE("{}", Json); + } + else + { + Result.ThrowError("VFS info fetch failed"sv); + + return 1; + } + } + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h new file mode 100644 index 000000000..35546c9b6 --- /dev/null +++ b/src/zen/cmds/vfs_cmd.h @@ -0,0 +1,26 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +class VfsCommand : public ZenCmdBase +{ +public: + VfsCommand(); + ~VfsCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"vfs", "Manage virtual file system"}; + + std::string m_Verb; + std::string m_HostName; + std::string m_MountPath; +}; + +} // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 5530b57b6..db67fbae0 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -20,6 +20,7 @@ #include "cmds/top.h" #include "cmds/up.h" #include "cmds/version.h" +#include "cmds/vfs_cmd.h" #include <zencore/filesystem.h> #include <zencore/logging.h> @@ -230,6 +231,7 @@ main(int argc, char** argv) UpCommand UpCmd; AttachCommand AttachCmd; VersionCommand VersionCmd; + VfsCommand VfsCmd; #if ZEN_WITH_TESTS RunTestsCommand RunTestsCmd; #endif @@ -279,6 +281,7 @@ main(int argc, char** argv) {"up", &UpCmd, "Bring zen server up"}, {"attach", &AttachCmd, "Add a sponsor process to a running zen service"}, {"version", &VersionCmd, "Get zen server version"}, + {"vfs", &VfsCmd, "Manage virtual file system"}, #if ZEN_WITH_TESTS {"runtests", &RunTestsCmd, "Run zen tests"}, #endif diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index b1ec14a37..e9f7ba72c 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -92,7 +92,7 @@ CreateDirectories(const wchar_t* Dir) // behind static bool -WipeDirectory(const wchar_t* DirPath) +WipeDirectory(const wchar_t* DirPath, bool KeepDotFiles) { ExtendableWideStringBuilder<128> Pattern; Pattern.Append(DirPath); @@ -107,7 +107,7 @@ WipeDirectory(const wchar_t* DirPath) { do { - bool IsRegular = true; + bool AttemptDelete = true; if (FindData.cFileName[0] == L'.') { @@ -115,16 +115,21 @@ WipeDirectory(const wchar_t* DirPath) { if (FindData.cFileName[2] == L'\0') { - IsRegular = false; + AttemptDelete = false; } } else if (FindData.cFileName[1] == L'\0') { - IsRegular = false; + AttemptDelete = false; + } + + if (KeepDotFiles) + { + AttemptDelete = false; } } - if (IsRegular) + if (AttemptDelete) { ExtendableWideStringBuilder<128> Path; Path.Append(DirPath); @@ -188,7 +193,8 @@ WipeDirectory(const wchar_t* DirPath) bool DeleteDirectories(const wchar_t* DirPath) { - return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE; + const bool KeepDotFiles = false; + return WipeDirectory(DirPath, KeepDotFiles) && RemoveDirectoryW(DirPath) == TRUE; } bool @@ -196,7 +202,20 @@ CleanDirectory(const wchar_t* DirPath) { if (std::filesystem::exists(DirPath)) { - return WipeDirectory(DirPath); + const bool KeepDotFiles = false; + + return WipeDirectory(DirPath, KeepDotFiles); + } + + return CreateDirectories(DirPath); +} + +bool +CleanDirectory(const wchar_t* DirPath, bool KeepDotFiles) +{ + if (std::filesystem::exists(DirPath)) + { + return WipeDirectory(DirPath, KeepDotFiles); } return CreateDirectories(DirPath); @@ -262,6 +281,19 @@ CleanDirectory(const std::filesystem::path& Dir) #endif } +bool +CleanDirectoryExceptDotFiles(const std::filesystem::path& Dir) +{ +#if ZEN_PLATFORM_WINDOWS + const bool KeepDotFiles = true; + return CleanDirectory(Dir.c_str(), KeepDotFiles); +#else + ZEN_UNUSED(Dir); + + ZEN_NOT_IMPLEMENTED(); +#endif +} + ////////////////////////////////////////////////////////////////////////// bool @@ -950,7 +982,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr if (FAILED(hRes)) { - ThrowSystemException(hRes, "Failed to open handle to volume root"); + ThrowSystemException(hRes, fmt::format("Failed to open handle to '{}'", RootDir)); } while (Continue) @@ -1063,6 +1095,33 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr closedir(Dir); #endif // ZEN_PLATFORM_WINDOWS } + +std::filesystem::path +CanonicalPath(std::filesystem::path InPath, std::error_code& Ec) +{ + ZEN_UNUSED(Ec); + +#if ZEN_PLATFORM_WINDOWS + windows::FileHandle Handle; + HRESULT hRes = Handle.Create(InPath.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS); + + if (FAILED(hRes)) + { + Ec = MakeErrorCodeFromLastError(); + + return {}; + } + + return PathFromHandle(Handle, Ec); +#else + return InPath; +#endif +} + std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec) { diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index a78edd16b..075188993 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -30,6 +30,10 @@ ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); */ ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir); + /** Map native file handle to a path */ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle); @@ -38,6 +42,10 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle); */ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec); +/** Get canonical path name from a generic path + */ +ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); + /** Query file size from native file handle */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp index 7adf07350..9e6f86d79 100644 --- a/src/zenserver/cache/cachedisklayer.cpp +++ b/src/zenserver/cache/cachedisklayer.cpp @@ -1620,6 +1620,19 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilt } void +ZenCacheDiskLayer::CacheBucket::EnumerateBucketContents( + std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const +{ + RwLock::SharedLockScope _(m_IndexLock); + for (const auto& It : m_Index) + { + CacheValueDetails::ValueDetails Vd = GetValueDetails(It.first, It.second); + + Fn(It.first, Vd); + } +} + +void ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx) { ZEN_TRACE_CPU("Z$::Disk::CollectGarbage"); @@ -2130,6 +2143,18 @@ ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const return {}; } +void +ZenCacheDiskLayer::EnumerateBucketContents(std::string_view Bucket, + std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const +{ + RwLock::SharedLockScope _(m_Lock); + + if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end()) + { + It->second->EnumerateBucketContents(Fn); + } +} + CacheValueDetails::NamespaceDetails ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const { diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h index 127e194f1..fc4d8cd6f 100644 --- a/src/zenserver/cache/cachedisklayer.h +++ b/src/zenserver/cache/cachedisklayer.h @@ -124,6 +124,8 @@ public: Info GetInfo() const; std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const; + void EnumerateBucketContents(std::string_view Bucket, + std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const; CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const; @@ -150,6 +152,7 @@ private: uint64_t EntryCount() const; CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const; + void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const; private: const uint64_t MaxBlockSize = 1ull << 30; diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp index c8384d330..4499b05f7 100644 --- a/src/zenserver/cache/structuredcachestore.cpp +++ b/src/zenserver/cache/structuredcachestore.cpp @@ -120,6 +120,13 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket) return AnyDropped; } +void +ZenCacheNamespace::EnumerateBucketContents(std::string_view Bucket, + std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const +{ + m_DiskLayer.EnumerateBucketContents(Bucket, Fn); +} + bool ZenCacheNamespace::Drop() { @@ -502,6 +509,17 @@ ZenCacheStore::GetValueDetails(const std::string_view NamespaceFilter, return Details; } +void +ZenCacheStore::EnumerateBucketContents(std::string_view Namespace, + std::string_view Bucket, + std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>&& Fn) +{ + if (const ZenCacheNamespace* Ns = FindNamespace(Namespace)) + { + Ns->EnumerateBucketContents(Bucket, Fn); + } +} + ZenCacheNamespace* ZenCacheStore::GetNamespace(std::string_view Namespace) { diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h index 8c1f995a4..239efe68f 100644 --- a/src/zenserver/cache/structuredcachestore.h +++ b/src/zenserver/cache/structuredcachestore.h @@ -69,10 +69,14 @@ public: ZenCacheNamespace(GcManager& Gc, const std::filesystem::path& RootDir); ~ZenCacheNamespace(); - bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); - void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); + bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); + + bool DropBucket(std::string_view Bucket); + void EnumerateBucketContents(std::string_view Bucket, + std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const; + bool Drop(); - bool DropBucket(std::string_view Bucket); void Flush(); uint64_t DiskLayerThreshold() const { return m_DiskLayerSizeThreshold; } @@ -160,6 +164,10 @@ public: std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket); std::vector<std::string> GetNamespaces(); + void EnumerateBucketContents(std::string_view Namespace, + std::string_view Bucket, + std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>&& Fn); + private: const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const; ZenCacheNamespace* GetNamespace(std::string_view Namespace); diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index e5739418e..3643e8011 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -62,7 +62,7 @@ HttpObjectStoreService::Inititalize() [this](zen::HttpRouterRequest& Request) { const std::string BucketName = Request.GetCapture(1); - StringBuilder<1024> Json; + ExtendableStringBuilder<1024> Json; { CbObjectWriter Writer; Writer.BeginArray("distributions"); diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 9be600e4e..1ad4403f4 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -691,6 +691,45 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId) return {}; } +std::vector<ProjectStore::Oplog::ChunkInfo> +ProjectStore::Oplog::GetAllChunksInfo() +{ + // First just capture all the chunk ids + + std::vector<ChunkInfo> InfoArray; + + { + RwLock::SharedLockScope _(m_OplogLock); + + if (m_Storage) + { + const size_t NumEntries = m_FileMap.size() + m_ChunkMap.size(); + + InfoArray.reserve(NumEntries); + + for (const auto& Kv : m_FileMap) + { + InfoArray.push_back({.ChunkId = Kv.first}); + } + + for (const auto& Kv : m_ChunkMap) + { + InfoArray.push_back({.ChunkId = Kv.first}); + } + } + } + + for (ChunkInfo& Info : InfoArray) + { + if (IoBuffer Chunk = FindChunk(Info.ChunkId)) + { + Info.ChunkSize = Chunk.GetSize(); + } + } + + return InfoArray; +} + void ProjectStore::Oplog::IterateFileMap( std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn) @@ -2056,6 +2095,49 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::strin } std::pair<HttpResponseCode, std::string> +ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload) +{ + ZEN_TRACE_CPU("ProjectStore::GetProjectChunks"); + + using namespace std::literals; + + Ref<ProjectStore::Project> Project = OpenProject(ProjectId); + if (!Project) + { + return {HttpResponseCode::NotFound, fmt::format("unknown project '{}'", ProjectId)}; + } + Project->TouchProject(); + + ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); + if (!FoundLog) + { + return {HttpResponseCode::NotFound, fmt::format("unknown oplog '{}/{}'", ProjectId, OplogId)}; + } + Project->TouchOplog(OplogId); + + std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfo = FoundLog->GetAllChunksInfo(); + + CbObjectWriter Response; + + Response.BeginArray("chunks"sv); + for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo) + { + Response << Info.ChunkId; + } + Response.EndArray(); + + Response.BeginArray("sizes"sv); + for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo) + { + Response << Info.ChunkSize; + } + Response.EndArray(); + + OutPayload = Response.Save(); + return {HttpResponseCode::OK, {}}; +} + +std::pair<HttpResponseCode, std::string> ProjectStore::GetChunkInfo(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view ChunkId, @@ -2120,6 +2202,25 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, ZenContentType AcceptType, IoBuffer& OutChunk) { + if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) + { + return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)}; + } + + const Oid Obj = Oid::FromHexString(ChunkId); + + return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk); +} + +std::pair<HttpResponseCode, std::string> +ProjectStore::GetChunkRange(const std::string_view ProjectId, + const std::string_view OplogId, + Oid ChunkId, + uint64_t Offset, + uint64_t Size, + ZenContentType AcceptType, + IoBuffer& OutChunk) +{ bool IsOffset = Offset != 0 || Size != ~(0ull); Ref<ProjectStore::Project> Project = OpenProject(ProjectId); @@ -2136,14 +2237,7 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId, } Project->TouchOplog(OplogId); - if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) - { - return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)}; - } - - const Oid Obj = Oid::FromHexString(ChunkId); - - IoBuffer Chunk = FoundLog->FindChunk(Obj); + IoBuffer Chunk = FoundLog->FindChunk(ChunkId); if (!Chunk) { return {HttpResponseCode::NotFound, {}}; diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index aa84d04ca..9c0f8790d 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -85,6 +85,13 @@ public: void Write(); void Update(const std::filesystem::path& MarkerPath); + struct ChunkInfo + { + Oid ChunkId; + uint64_t ChunkSize; + }; + + std::vector<ChunkInfo> GetAllChunksInfo(); void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn); void IterateOplog(std::function<void(CbObject)>&& Fn); void IterateOplogWithKey(std::function<void(int, const Oid&, CbObject)>&& Fn); @@ -287,12 +294,22 @@ public: const std::string_view OplogId, bool FilterClient, CbObject& OutPayload); + std::pair<HttpResponseCode, std::string> GetProjectChunks(const std::string_view ProjectId, + const std::string_view OplogId, + CbObject& OutPayload); std::pair<HttpResponseCode, std::string> GetChunkInfo(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view ChunkId, CbObject& OutPayload); std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId, const std::string_view OplogId, + const Oid ChunkId, + uint64_t Offset, + uint64_t Size, + ZenContentType AcceptType, + IoBuffer& OutChunk); + std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId, + const std::string_view OplogId, const std::string_view ChunkId, uint64_t Offset, uint64_t Size, diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp new file mode 100644 index 000000000..f74000900 --- /dev/null +++ b/src/zenserver/vfs/vfsimpl.cpp @@ -0,0 +1,457 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "vfsimpl.h" +#include "vfsservice.h" + +#include "cache/structuredcachestore.h" +#include "projectstore/projectstore.h" + +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zenvfs/projfs.h> +#include <zenvfs/vfs.h> + +#include <memory> + +#if ZEN_WITH_VFS + +namespace zen { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +VfsOplogDataSource::VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore) +: m_ProjectId(ProjectId) +, m_OplogId(OplogId) +, m_ProjectStore(std::move(InProjectStore)) +{ +} + +void +VfsOplogDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) +{ + ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount); +} + +void +VfsOplogDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) +{ + IoBuffer ChunkBuffer; + auto Result = + m_ProjectStore->GetChunkRange(m_ProjectId, m_OplogId, ChunkId, 0, ~0ull, ZenContentType::kCompressedBinary, /* out */ ChunkBuffer); + + if (Result.first == HttpResponseCode::OK) + { + const uint8_t* SourceBuffer = reinterpret_cast<const uint8_t*>(ChunkBuffer.GetData()); + uint64_t AvailableBufferBytes = ChunkBuffer.GetSize(); + + ZEN_ASSERT(AvailableBufferBytes >= ByteOffset); + AvailableBufferBytes -= ByteOffset; + SourceBuffer += ByteOffset; + + ZEN_ASSERT(AvailableBufferBytes >= ByteCount); + memcpy(Buffer, SourceBuffer, ByteCount); + } +} + +void +VfsOplogDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) +{ + // This should never be called + ZEN_UNUSED(NodePath, DirNode); +} + +////////////////////////////////////////////////////////////////////////// + +VfsCacheDataSource::VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore) +: m_NamespaceId(NamespaceId) +, m_BucketId(BucketId) +, m_CacheStore(std::move(InCacheStore)) +{ +} + +VfsCacheDataSource::~VfsCacheDataSource() +{ +} + +void +VfsCacheDataSource::ReadNamedData(std::string_view Name, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) +{ + if (auto DotIndex = Name.find_first_of('.'); DotIndex != std::string_view::npos) + { + Name = Name.substr(0, DotIndex); + } + + IoHash HashKey = IoHash::FromHexString(Name); + + CacheRequestContext CacheContext{}; + + ZenCacheValue Value; + if (m_CacheStore->Get(CacheContext, m_NamespaceId, m_BucketId, HashKey, /* out */ Value)) + { + // TODO bounds check! + auto DataPtr = reinterpret_cast<const uint8_t*>(Value.Value.GetData()) + ByteOffset; + + memcpy(Buffer, DataPtr, ByteCount); + + return; + } +} + +void +VfsCacheDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) +{ + ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount); +} + +void +VfsCacheDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) +{ + ZEN_UNUSED(NodePath, DirNode); +} + +////////////////////////////////////////////////////////////////////////// + +VfsService::Impl::Impl() +{ +} + +VfsService::Impl::~Impl() +{ + Unmount(); +} + +void +VfsService::Impl::Mount(std::string_view MountPoint) +{ + ZEN_INFO("VFS mount requested at '{}'", MountPoint); + +# if ZEN_PLATFORM_WINDOWS + if (!IsProjFsAvailable()) + { + throw std::runtime_error("Projected File System component not available"); + } +# endif + + if (!m_MountpointPath.empty()) + { + throw std::runtime_error("VFS already mounted"); + } + + m_MountpointPath = MountPoint; + + RefreshVfs(); +} + +void +VfsService::Impl::Unmount() +{ + if (m_MountpointPath.empty()) + { + return; + } + + ZEN_INFO("unmounting VFS from '{}'", m_MountpointPath); + + m_MountpointPath.clear(); + + RefreshVfs(); +} + +void +VfsService::Impl::AddService(Ref<ProjectStore>&& Ps) +{ + m_ProjectStore = std::move(Ps); + + RefreshVfs(); +} + +void +VfsService::Impl::AddService(Ref<ZenCacheStore>&& Z$) +{ + m_ZenCacheStore = std::move(Z$); + + RefreshVfs(); +} + +void +VfsService::Impl::RefreshVfs() +{ + if (m_VfsHost && m_MountpointPath.empty()) + { + m_VfsHost->RequestStop(); + m_VfsThread.join(); + m_VfsHost.reset(); + m_VfsThreadRunning.Reset(); + m_VfsDataSource = nullptr; + + return; + } + + if (!m_VfsHost && !m_MountpointPath.empty()) + { + m_VfsThread = std::thread(&VfsService::Impl::VfsThread, this); + m_VfsThreadRunning.Wait(); + + // At this stage, m_VfsHost should be initialized + + ZEN_ASSERT(m_VfsHost); + } + + if (m_ProjectStore && m_VfsHost) + { + if (!m_VfsDataSource) + { + m_VfsDataSource = new VfsServiceDataSource(this); + } + + m_VfsHost->AddMount("projects"sv, m_VfsDataSource); + } + + if (m_ZenCacheStore && m_VfsHost) + { + if (!m_VfsDataSource) + { + m_VfsDataSource = new VfsServiceDataSource(this); + } + + m_VfsHost->AddMount("ddc_cache"sv, m_VfsDataSource); + } +} + +void +VfsService::Impl::VfsThread() +{ + SetCurrentThreadName("VFS"); + + ZEN_INFO("VFS service thread now RUNNING"); + + try + { + m_VfsHost = std::make_unique<VfsHost>(m_MountpointPath); + m_VfsHost->Initialize(); + + m_VfsThreadRunning.Set(); + m_VfsHost->Run(); + } + catch (std::exception& Ex) + { + ZEN_WARN("exception caught in VFS thread: {}", Ex.what()); + + m_VfsThreadException = std::current_exception(); + } + + if (m_VfsHost) + { + m_VfsHost->Cleanup(); + } + + ZEN_INFO("VFS service thread now EXITING"); +} + +////////////////////////////////////////////////////////////////////////// + +Ref<VfsOplogDataSource> +VfsServiceDataSource::GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId) +{ + ExtendableStringBuilder<256> Key; + Key << ProjectId << "." << OplogId; + std::string StdKey{Key}; + + RwLock::ExclusiveLockScope _(m_Lock); + + if (auto It = m_OplogSourceMap.find(StdKey); It == m_OplogSourceMap.end()) + { + Ref<VfsOplogDataSource> NewSource{new VfsOplogDataSource(ProjectId, OplogId, m_VfsImpl->m_ProjectStore)}; + m_OplogSourceMap[StdKey] = NewSource; + return NewSource; + } + else + { + return It->second; + } +} + +Ref<VfsCacheDataSource> +VfsServiceDataSource::GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId) +{ + ExtendableStringBuilder<256> Key; + Key << NamespaceId << "." << BucketId; + std::string StdKey{Key}; + + RwLock::ExclusiveLockScope _(m_Lock); + + if (auto It = m_CacheSourceMap.find(StdKey); It == m_CacheSourceMap.end()) + { + Ref<VfsCacheDataSource> NewSource{new VfsCacheDataSource(NamespaceId, BucketId, m_VfsImpl->m_ZenCacheStore)}; + m_CacheSourceMap[StdKey] = NewSource; + return NewSource; + } + else + { + return It->second; + } +} + +void +VfsServiceDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) +{ + ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount); +} + +void +VfsServiceDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) +{ + ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount); +} + +void +VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) +{ + if (NodePath == "projects"sv) + { + // Project enumeration + + m_VfsImpl->m_ProjectStore->DiscoverProjects(); + + m_VfsImpl->m_ProjectStore->IterateProjects( + [&](ProjectStore::Project& Project) { DirNode.AddVirtualNode(Project.Identifier, m_VfsImpl->m_VfsDataSource); }); + } + else if (NodePath.starts_with("projects\\"sv)) + { + std::string_view ProjectId{NodePath}; + ProjectId = ProjectId.substr(9); // Skip "projects\" + + if (std::string_view::size_type SlashOffset = ProjectId.find_first_of('\\'); SlashOffset == std::string_view::npos) + { + Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId); + + if (!Project) + { + // No such project found? + + return; + } + + // Oplog enumeration + + std::vector<std::string> Oplogs = Project->ScanForOplogs(); + + for (auto& Oplog : Oplogs) + { + DirNode.AddVirtualNode(Oplog, m_VfsImpl->m_VfsDataSource); + } + } + else + { + std::string_view OplogId = ProjectId.substr(SlashOffset + 1); + ProjectId = ProjectId.substr(0, SlashOffset); + + Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId); + + if (!Project) + { + // No such project found? + + return; + } + + // Oplog contents enumeration + + if (ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId)) + { + Ref<VfsOplogDataSource> DataSource = GetOplogDataSource(ProjectId, OplogId); + + // Get metadata for all chunks + std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfos = Oplog->GetAllChunksInfo(); + + std::unordered_map<zen::Oid, uint64_t> ChunkSizes; + + for (const auto& Ci : ChunkInfos) + { + ChunkSizes[Ci.ChunkId] = Ci.ChunkSize; + } + + auto EmitFilesForDataArray = [&](zen::CbArrayView DataArray) { + for (auto DataIter : DataArray) + { + if (zen::CbObjectView Data = DataIter.AsObjectView()) + { + std::string_view FileName = Data["filename"sv].AsString(); + zen::Oid ChunkId = Data["id"sv].AsObjectId(); + + if (auto FindIt = ChunkSizes.find(ChunkId); FindIt != ChunkSizes.end()) + { + DirNode.AddFileNode(FileName, FindIt->second /* file size */, ChunkId, DataSource); + } + else + { + ZEN_WARN("no chunk metadata found for chunk {} (file: '{}')", ChunkId, FileName); + } + } + } + }; + + Oplog->IterateOplog([&](CbObject Op) { + EmitFilesForDataArray(Op["packagedata"sv].AsArrayView()); + EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView()); + }); + + DirNode.AddFileNode("stats.json", 42, Oid::Zero); + } + } + } + else if (NodePath == "ddc_cache"sv) + { + // Namespace enumeration + + std::vector<std::string> Namespaces = m_VfsImpl->m_ZenCacheStore->GetNamespaces(); + + for (auto& Namespace : Namespaces) + { + DirNode.AddVirtualNode(Namespace, m_VfsImpl->m_VfsDataSource); + } + } + else if (NodePath.starts_with("ddc_cache\\"sv)) + { + std::string_view NamespaceId{NodePath}; + NamespaceId = NamespaceId.substr(10); // Skip "ddc_cache\" + + auto& Cache = m_VfsImpl->m_ZenCacheStore; + + if (std::string_view::size_type SlashOffset = NamespaceId.find_first_of('\\'); SlashOffset == std::string_view::npos) + { + // Bucket enumeration + + if (auto NsInfo = Cache->GetNamespaceInfo(NamespaceId)) + { + for (auto& BucketName : NsInfo->BucketNames) + { + DirNode.AddVirtualNode(BucketName, m_VfsImpl->m_VfsDataSource); + } + } + } + else + { + // Bucket contents enumeration + + std::string_view BucketId = NamespaceId.substr(SlashOffset + 1); + NamespaceId = NamespaceId.substr(0, SlashOffset); + + Ref<VfsCacheDataSource> DataSource = GetCacheDataSource(NamespaceId, BucketId); + + auto Enumerator = [&](const IoHash& Key, const CacheValueDetails::ValueDetails& Details) { + ExtendableStringBuilder<64> KeyString; + Key.ToHexString(KeyString); + KeyString.Append(".udd"); + DirNode.AddFileNode(KeyString, Details.Size, Oid::Zero, DataSource); + }; + + Cache->EnumerateBucketContents(NamespaceId, BucketId, Enumerator); + } + } +} + +} // namespace zen +#endif diff --git a/src/zenserver/vfs/vfsimpl.h b/src/zenserver/vfs/vfsimpl.h new file mode 100644 index 000000000..9e4fdfe99 --- /dev/null +++ b/src/zenserver/vfs/vfsimpl.h @@ -0,0 +1,100 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "vfsservice.h" + +#include "projectstore/projectstore.h" + +#include <zencore/logging.h> +#include <zenvfs/vfs.h> + +#if ZEN_WITH_VFS + +# include <memory> + +namespace zen { + +struct VfsOplogDataSource : public VfsTreeDataSource +{ + VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore); + + virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; + virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; + virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override; + +private: + std::string m_ProjectId; + std::string m_OplogId; + Ref<ProjectStore> m_ProjectStore; +}; + +struct VfsCacheDataSource : public VfsTreeDataSource +{ + VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore); + ~VfsCacheDataSource(); + + virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; + virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; + virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override; + +private: + std::string m_NamespaceId; + std::string m_BucketId; + Ref<ZenCacheStore> m_CacheStore; +}; + +struct VfsServiceDataSource : public VfsTreeDataSource +{ + VfsServiceDataSource(VfsService::Impl* VfsImpl) : m_VfsImpl(VfsImpl) {} + + virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; + virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; + virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override; + +private: + VfsService::Impl* m_VfsImpl = nullptr; + + RwLock m_Lock; + std::unordered_map<std::string, Ref<VfsOplogDataSource>> m_OplogSourceMap; + std::unordered_map<std::string, Ref<VfsCacheDataSource>> m_CacheSourceMap; + + Ref<VfsOplogDataSource> GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId); + Ref<VfsCacheDataSource> GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId); +}; + +////////////////////////////////////////////////////////////////////////// + +struct VfsService::Impl +{ + Impl(); + ~Impl(); + + void Mount(std::string_view MountPoint); + void Unmount(); + void AddService(Ref<ProjectStore>&&); + void AddService(Ref<ZenCacheStore>&&); + + inline std::string GetMountpointPath() { return m_MountpointPath; } + inline bool IsVfsRunning() const { return !!m_VfsHost.get(); } + +private: + Ref<ProjectStore> m_ProjectStore; + Ref<ZenCacheStore> m_ZenCacheStore; + Ref<VfsServiceDataSource> m_VfsDataSource; + std::string m_MountpointPath; + + std::unique_ptr<VfsHost> m_VfsHost; + std::thread m_VfsThread; + Event m_VfsThreadRunning; + std::exception_ptr m_VfsThreadException; + + void RefreshVfs(); + void VfsThread(); + + friend struct VfsServiceDataSource; +}; + +} // namespace zen + +#endif diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp new file mode 100644 index 000000000..c53682d93 --- /dev/null +++ b/src/zenserver/vfs/vfsservice.cpp @@ -0,0 +1,217 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "vfsservice.h" +#include "vfsimpl.h" + +#include <zencore/compactbinarybuilder.h> + +namespace zen { + +using namespace std::literals; + +#if ZEN_WITH_VFS + +////////////////////////////////////////////////////////////////////////// + +bool +GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) +{ + IoBuffer Payload = HttpReq.ReadPayload(); + HttpContentType PayloadContentType = HttpReq.RequestContentType(); + + switch (PayloadContentType) + { + case HttpContentType::kJSON: + case HttpContentType::kUnknownContentType: + case HttpContentType::kText: + { + std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize()); + Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); + if (!Cb) + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected JSON format"); + return false; + } + } + break; + case HttpContentType::kCbObject: + Cb = LoadCompactBinaryObject(Payload); + if (!Cb) + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected compact binary format"); + return false; + } + break; + default: + HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + return false; + } + + return true; +} + +////////////////////////////////////////////////////////////////////////// +// +// to test: +// +// echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:1337/vfs --data-binary @- +// echo {"method": "unmount"} | curl.exe http://localhost:1337/vfs --data-binary @- + +VfsService::VfsService() +{ + m_Impl = new Impl; + + m_Router.RegisterRoute( + "info", + [&](HttpRouterRequest& Request) { + CbObjectWriter Cbo; + Cbo << "running" << m_Impl->IsVfsRunning(); + Cbo << "rootpath" << m_Impl->GetMountpointPath(); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "", + [&](HttpRouterRequest& Req) { + CbObject Payload; + + if (!GetContentAsCbObject(Req.ServerRequest(), Payload)) + return; + + std::string_view RpcName = Payload["method"sv].AsString(); + + if (RpcName == "mount"sv) + { + CbObjectView Params = Payload["params"sv].AsObjectView(); + std::string_view Mountpath = Params["path"sv].AsString(); + + if (Mountpath.empty()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "no path specified"); + } + + if (m_Impl->IsVfsRunning()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS already mounted"); + } + + try + { + m_Impl->Mount(Mountpath); + } + catch (std::exception& Ex) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); + } + + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + } + else if (RpcName == "unmount"sv) + { + if (!m_Impl->IsVfsRunning()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS not active"); + } + + try + { + m_Impl->Unmount(); + } + catch (std::exception& Ex) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); + } + + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + } + else + { + Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "unknown RPC"sv); + } + }, + HttpVerb::kPost); +} + +VfsService::~VfsService() +{ + delete m_Impl; +} + +void +VfsService::Mount(std::string_view MountPoint) +{ + m_Impl->Mount(MountPoint); +} + +void +VfsService::Unmount() +{ + m_Impl->Unmount(); +} + +void +VfsService::AddService(Ref<ProjectStore>&& Ps) +{ + m_Impl->AddService(std::move(Ps)); +} + +void +VfsService::AddService(Ref<ZenCacheStore>&& Z$) +{ + m_Impl->AddService(std::move(Z$)); +} + +#else + +VfsService::VfsService() +{ +} + +VfsService::~VfsService() +{ +} + +void +VfsService::Mount(std::string_view MountPoint) +{ + ZEN_UNUSED(MountPoint); +} + +void +VfsService::Unmount() +{ +} + +void +VfsService::AddService(Ref<ProjectStore>&& Ps) +{ + ZEN_UNUSED(Ps); +} + +void +VfsService::AddService(Ref<ZenCacheStore>&& Z$) +{ + ZEN_UNUSED(Z$); +} + +#endif + +const char* +VfsService::BaseUri() const +{ + return "/vfs/"; +} + +void +VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest) +{ + m_Router.HandleRequest(HttpServiceRequest); +} + +} // namespace zen diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h new file mode 100644 index 000000000..9510cfcda --- /dev/null +++ b/src/zenserver/vfs/vfsservice.h @@ -0,0 +1,52 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/refcount.h> +#include <zenhttp/httpserver.h> +#include <zenvfs/vfs.h> + +#include <memory> + +namespace zen { + +class ProjectStore; +class ZenCacheStore; + +/** Virtual File System service + + Implements support for exposing data via a virtual file system interface. Currently + this is primarily used to surface various data stored in the local storage service + to users for debugging and exploration purposes. + + Currently, it surfaces information from the structured cache service and from the + project store. + + */ + +class VfsService : public HttpService +{ +public: + VfsService(); + ~VfsService(); + + void Mount(std::string_view MountPoint); + void Unmount(); + + void AddService(Ref<ProjectStore>&&); + void AddService(Ref<ZenCacheStore>&&); + +protected: + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override; + +private: + struct Impl; + Impl* m_Impl = nullptr; + + HttpRequestRouter m_Router; + + friend struct VfsServiceDataSource; +}; + +} // namespace zen diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index a2d02baae..66bfe4ada 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -3,6 +3,7 @@ target("zenserver") set_kind("binary") add_deps("zencore", "zenhttp", "zenstore", "zenutil") + add_deps("zenvfs") add_headerfiles("**.h") add_files("**.cpp") add_files("zenserver.cpp", {unity_ignored = true }) @@ -19,6 +20,8 @@ target("zenserver") add_ldflags("/LTCG") add_files("zenserver.rc") add_cxxflags("/bigobj") + add_links("delayimp", "projectedfslib") + add_ldflags("/delayload:ProjectedFSLib.dll") else remove_files("windows/**") end diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 988f72273..1f37e336f 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -125,6 +125,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "projectstore/httpprojectstore.h" #include "projectstore/projectstore.h" #include "upstream/upstream.h" +#include "vfs/vfsservice.h" #define ZEN_APP_NAME "Zen store" @@ -419,6 +420,11 @@ public: m_Http->RegisterService(*m_ObjStoreService); } + m_VfsService = std::make_unique<VfsService>(); + m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore)); + m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore)); + m_Http->RegisterService(*m_VfsService); + ZEN_INFO("initializing GC, enabled '{}', interval {}s", ServerOptions.GcConfig.Enabled, ServerOptions.GcConfig.IntervalSeconds); zen::GcSchedulerConfig GcConfig{ .RootDirectory = m_DataRoot / "gc", @@ -750,6 +756,7 @@ private: #endif // ZEN_WITH_COMPUTE_SERVICES std::unique_ptr<zen::HttpFrontendService> m_FrontendService; std::unique_ptr<zen::HttpObjectStoreService> m_ObjStoreService; + std::unique_ptr<zen::VfsService> m_VfsService; std::unique_ptr<JobQueue> m_JobQueue; std::unique_ptr<zen::HttpAdminService> m_AdminService; diff --git a/src/zenvfs/include/zenvfs/projfs.h b/src/zenvfs/include/zenvfs/projfs.h new file mode 100644 index 000000000..8a89c2835 --- /dev/null +++ b/src/zenvfs/include/zenvfs/projfs.h @@ -0,0 +1,14 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenvfs.h" + +#if ZEN_WITH_VFS + +namespace zen { + +bool IsProjFsAvailable(); + +} // namespace zen +#endif diff --git a/src/zenvfs/include/zenvfs/vfs.h b/src/zenvfs/include/zenvfs/vfs.h new file mode 100644 index 000000000..412970860 --- /dev/null +++ b/src/zenvfs/include/zenvfs/vfs.h @@ -0,0 +1,89 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenvfs.h" + +#if ZEN_WITH_VFS + +# include <zencore/zencore.h> +# include <zencore/refcount.h> +# include <zencore/uid.h> + +# include <string_view> +# include <memory> +# include <vector> + +namespace zen { + +struct VfsProvider; + +////////////////////////////////////////////////////////////////////////// + +struct VfsTreeNode; + +struct VfsTreeDataSource : public RefCounted +{ + virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) = 0; + virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) = 0; + virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) = 0; +}; + +struct VfsTreeNode +{ + inline VfsTreeNode(VfsTreeNode* InParent) : ParentNode(InParent) {} + + const std::string_view GetName() const { return Name; } + uint64_t GetFileSize() const { return FileSize; } + VfsTreeDataSource* GetDataSource() const { return DataSource.Get(); } + const Oid& GetChunkId() const { return ChunkId; } + + const std::string GetPath() const; + + VfsTreeNode* FindNode(std::wstring_view NodeName); + VfsTreeNode* FindNode(std::string_view NodeName); + + void AddVirtualNode(std::string_view NodeName, Ref<VfsTreeDataSource>&& DataSource); + void AddFileNode(std::string_view NodeName, uint64_t InFileSize, Oid InChunkId, Ref<VfsTreeDataSource> DataSource = {}); + void NormalizeTree(); + bool IsDirectory() const; + void PopulateVirtualNode(); + + const std::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() const { return ChildArray; } + std::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() { return ChildArray; } + +private: + std::string Name; + uint64_t FileSize = 0; + Oid ChunkId = Oid::Zero; + Ref<VfsTreeDataSource> DataSource; + + inline static const uint64_t DirectoryMarker = ~uint64_t(0); + std::vector<std::unique_ptr<VfsTreeNode>> ChildArray; + VfsTreeNode* ParentNode = nullptr; + + VfsTreeNode* AddNewChildNode() { return ChildArray.emplace_back(std::make_unique<VfsTreeNode>(this)).get(); } + + void GetPath(StringBuilderBase& Path) const; +}; + +////////////////////////////////////////////////////////////////////////// + +class VfsHost +{ +public: + VfsHost(std::string_view VfsRootPath); + ~VfsHost(); + + void Initialize(); + void AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource); + void Run(); + void RequestStop(); + void Cleanup(); + +private: + VfsProvider* m_Provider = nullptr; +}; + +} // namespace zen +#endif diff --git a/src/zenvfs/include/zenvfs/zenvfs.h b/src/zenvfs/include/zenvfs/zenvfs.h new file mode 100644 index 000000000..c70368a1d --- /dev/null +++ b/src/zenvfs/include/zenvfs/zenvfs.h @@ -0,0 +1,9 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#ifndef ZEN_WITH_VFS +# define ZEN_WITH_VFS ZEN_PLATFORM_WINDOWS +#endif diff --git a/src/zenvfs/projfs.cpp b/src/zenvfs/projfs.cpp new file mode 100644 index 000000000..5d74337b4 --- /dev/null +++ b/src/zenvfs/projfs.cpp @@ -0,0 +1,38 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenvfs/projfs.h" + +#if ZEN_WITH_VFS +# if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> + +namespace zen { + +bool +IsProjFsAvailable() +{ + HMODULE hMod = LoadLibraryA("projectedfslib.dll"); + + if (hMod) + { + FreeLibrary(hMod); + + return true; + } + + return false; +} + +} // namespace zen +# else +namespace zen { + +bool +IsProjFsAvailable() +{ + return false; +} + +} // namespace zen +# endif +#endif diff --git a/src/zenvfs/projfsproviderinterface.cpp b/src/zenvfs/projfsproviderinterface.cpp new file mode 100644 index 000000000..ca4b91382 --- /dev/null +++ b/src/zenvfs/projfsproviderinterface.cpp @@ -0,0 +1,98 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "projfsproviderinterface.h" + +#if ZEN_WITH_VFS + +namespace zen { + +HRESULT +ProjFsProviderInterface::Notify(const PRJ_CALLBACK_DATA* CallbackData, + BOOLEAN IsDirectory, + PRJ_NOTIFICATION NotificationType, + PCWSTR DestinationFileName, + PRJ_NOTIFICATION_PARAMETERS* NotificationParameters) +{ + ZEN_UNUSED(CallbackData, IsDirectory, NotificationType, DestinationFileName, NotificationParameters); + return S_OK; +} + +HRESULT +ProjFsProviderInterface::QueryFileName(const PRJ_CALLBACK_DATA* CallbackData) +{ + ZEN_UNUSED(CallbackData); + return S_OK; +} + +void +ProjFsProviderInterface::CancelCommand(const PRJ_CALLBACK_DATA* CallbackData) +{ + ZEN_UNUSED(CallbackData); +} + +////////////////////////////////////////////////////////////////////////// +// +// Forwarding functions for all ProjFS callbacks - these simply call the +// corresponding member function on the active provider +// + +HRESULT +ProjFsProviderInterface::StartDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) +{ + return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->StartDirEnum(CallbackData, EnumerationId); +} + +HRESULT +ProjFsProviderInterface::EndDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) +{ + return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->EndDirEnum(CallbackData, EnumerationId); +} + +HRESULT +ProjFsProviderInterface::GetDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ const GUID* EnumerationId, + _In_opt_ PCWSTR SearchExpression, + _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) +{ + return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext) + ->GetDirEnum(CallbackData, EnumerationId, SearchExpression, DirEntryBufferHandle); +} + +HRESULT +ProjFsProviderInterface::GetPlaceholderInfoCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData) +{ + return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->GetPlaceholderInfo(CallbackData); +} + +HRESULT +ProjFsProviderInterface::GetFileDataCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) +{ + return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->GetFileData(CallbackData, ByteOffset, Length); +} + +HRESULT +ProjFsProviderInterface::NotificationCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ BOOLEAN IsDirectory, + _In_ PRJ_NOTIFICATION NotificationType, + _In_opt_ PCWSTR DestinationFileName, + _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters) +{ + return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext) + ->Notify(CallbackData, IsDirectory, NotificationType, DestinationFileName, NotificationParameters); +} + +HRESULT +ProjFsProviderInterface::QueryFileName_C(_In_ const PRJ_CALLBACK_DATA* CallbackData) +{ + return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->QueryFileName(CallbackData); +} + +void +ProjFsProviderInterface::CancelCommand_C(_In_ const PRJ_CALLBACK_DATA* CallbackData) +{ + return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->CancelCommand(CallbackData); +} + +} // namespace zen + +#endif diff --git a/src/zenvfs/projfsproviderinterface.h b/src/zenvfs/projfsproviderinterface.h new file mode 100644 index 000000000..33ea4b177 --- /dev/null +++ b/src/zenvfs/projfsproviderinterface.h @@ -0,0 +1,89 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenvfs/vfs.h> +#include <zenvfs/zenvfs.h> + +#if ZEN_WITH_VFS && ZEN_PLATFORM_WINDOWS + +ZEN_THIRD_PARTY_INCLUDES_START +# include <zencore/windows.h> +# include <projectedfslib.h> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +struct ProjFsProviderInterface +{ + virtual ~ProjFsProviderInterface() = default; + + // [Mandatory] Inform the provider a directory enumeration is starting. + // It corresponds to PRJ_START_DIRECTORY_ENUMERATION_CB in projectedfslib.h + virtual HRESULT StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) = 0; + + // [Mandatory] Inform the provider a directory enumeration is over. + // It corresponds to PRJ_END_DIRECTORY_ENUMERATION_CB in projectedfslib.h + virtual HRESULT EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) = 0; + + // [Mandatory] Request directory enumeration information from the provider. + // It corresponds to PRJ_GET_DIRECTORY_ENUMERATION_CB in projectedfslib.h + virtual HRESULT GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ const GUID* EnumerationId, + _In_opt_ PCWSTR SearchExpression, + _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) = 0; + + // [Mandatory] Request meta data information for a file or directory. + // It corresponds to PRJ_GET_PLACEHOLDER_INFO_CB in projectedfslib.h + virtual HRESULT GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData) = 0; + + // [Mandatory] Request the contents of a file's primary data stream. + // It corresponds to PRJ_GET_FILE_DATA_CB in projectedfslib.h + virtual HRESULT GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) = 0; + + // [Optional] Deliver notifications to the provider that it has specified + // it wishes to receive. It corresponds to PRJ_NOTIFICATION_CB in projectedfslib.h + virtual HRESULT Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ BOOLEAN IsDirectory, + _In_ PRJ_NOTIFICATION NotificationType, + _In_opt_ PCWSTR DestinationFileName, + _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters); + + /** This callback is optional. If the provider does not supply an implementation of this callback, ProjFS will invoke the provider's + directory enumeration callbacks to determine the existence of a file path in the provider's store. + + The provider should use PrjFileNameCompare as the comparison routine when searching its backing store for the specified file. + */ + virtual HRESULT QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData); + + virtual void CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData); + + // Setup + + virtual void Run() = 0; + + ////////////////////////////////////////////////////////////////////////// + // Forwarding functions + // + + static HRESULT StartDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId); + static HRESULT EndDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId); + static HRESULT GetDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ const GUID* EnumerationId, + _In_opt_ PCWSTR SearchExpression, + _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle); + static HRESULT GetPlaceholderInfoCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData); + static HRESULT GetFileDataCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length); + static HRESULT NotificationCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ BOOLEAN IsDirectory, + _In_ PRJ_NOTIFICATION NotificationType, + _In_opt_ PCWSTR DestinationFileName, + _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters); + static HRESULT QueryFileName_C(_In_ const PRJ_CALLBACK_DATA* CallbackData); + static void CancelCommand_C(_In_ const PRJ_CALLBACK_DATA* CallbackData); +}; + +} // namespace zen +#endif diff --git a/src/zenvfs/vfs.cpp b/src/zenvfs/vfs.cpp new file mode 100644 index 000000000..02dbc904c --- /dev/null +++ b/src/zenvfs/vfs.cpp @@ -0,0 +1,56 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenvfs/vfs.h" +#include "vfsprovider.h" + +#include <zencore/iohash.h> +#include <zencore/scopeguard.h> +#include <zencore/string.h> +#include <zencore/thread.h> + +#if ZEN_WITH_VFS + +namespace zen { + +VfsHost::VfsHost(std::string_view VfsRootPath) : m_Provider(new VfsProvider(VfsRootPath)) +{ +} + +VfsHost::~VfsHost() +{ + delete m_Provider; +} + +void +VfsHost::Initialize() +{ + m_Provider->Initialize(); +} + +void +VfsHost::Run() +{ + m_Provider->Run(); +} + +void +VfsHost::RequestStop() +{ + m_Provider->RequestStop(); +} + +void +VfsHost::Cleanup() +{ + m_Provider->Cleanup(); +} + +void +VfsHost::AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource) +{ + m_Provider->AddMount(Mountpoint, std::move(DataSource)); +} + +} // namespace zen + +#endif diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp new file mode 100644 index 000000000..a3cfe9d15 --- /dev/null +++ b/src/zenvfs/vfsprovider.cpp @@ -0,0 +1,794 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "vfsprovider.h" + +#if ZEN_WITH_VFS + +# include <zencore/filesystem.h> +# include <zencore/scopeguard.h> + +# include <system_error> + +namespace zen { + +bool +VfsTreeNode::IsDirectory() const +{ + return !GetChildren().empty() || FileSize == DirectoryMarker; +} + +VfsTreeNode* +VfsTreeNode::FindNode(std::wstring_view NodeName) +{ + if (auto SlashIndex = NodeName.find_first_of('\\'); SlashIndex != std::string_view::npos) + { + std::wstring_view DirName = NodeName.substr(0, SlashIndex); + + if (VfsTreeNode* DirNode = FindNode(DirName)) + { + return DirNode->FindNode(NodeName.substr(SlashIndex + 1)); + } + } + else + { + ExtendableWideStringBuilder<64> NodeNameZ; + NodeNameZ.Append(NodeName); + + PopulateVirtualNode(); + + for (auto& Child : GetChildren()) + { + ExtendableWideStringBuilder<64> ChildName; + Utf8ToWide(Child->Name, ChildName); + + if (PrjFileNameCompare(ChildName.c_str(), NodeNameZ.c_str()) == 0) + { + return Child.get(); + } + } + } + + return nullptr; +} + +VfsTreeNode* +VfsTreeNode::FindNode(std::string_view NodeName) +{ + for (auto& Child : GetChildren()) + { + if (NodeName == Child->Name) + { + return Child.get(); + } + } + + return nullptr; +} + +void +VfsTreeNode::PopulateVirtualNode() +{ + if (IsDirectory() && GetChildren().empty()) + { + // Populate this node via the data source + + DataSource->PopulateDirectory(GetPath(), *this); + NormalizeTree(); + } +} + +void +VfsTreeNode::AddFileNode(std::string_view NodeName, uint64_t InFileSize, Oid InChunkId, Ref<VfsTreeDataSource> InDataSource) +{ + auto SlashIndex = NodeName.find_first_of('/'); + + if (SlashIndex != std::string_view::npos) + { + const std::string_view DirName = NodeName.substr(0, SlashIndex); + const std::string_view ChildPath = NodeName.substr(SlashIndex + 1); + + VfsTreeNode* DirNode = FindNode(DirName); + + if (!DirNode) + { + VfsTreeNode* NewNode = AddNewChildNode(); + NewNode->Name = std::string(DirName); + + DirNode = NewNode; + } + + DirNode->AddFileNode(ChildPath, InFileSize, InChunkId, std::move(InDataSource)); + } + else + { + auto InitLeafNode = [&](VfsTreeNode& NewNode) { + NewNode.Name = std::string(NodeName); + NewNode.FileSize = InFileSize; + NewNode.ChunkId = InChunkId; + NewNode.DataSource = std::move(InDataSource); + }; + + for (auto& Child : GetChildren()) + { + // Already present, just update properties + + if (Child->Name == NodeName) + { + InitLeafNode(*Child); + + return; + } + } + + InitLeafNode(*AddNewChildNode()); + } +} + +void +VfsTreeNode::AddVirtualNode(std::string_view NodeName, Ref<VfsTreeDataSource>&& InDataSource) +{ + auto SlashIndex = NodeName.find_first_of('/'); + + if (SlashIndex != std::string_view::npos) + { + const std::string_view DirName = NodeName.substr(0, SlashIndex); + const std::string_view ChildPath = NodeName.substr(SlashIndex + 1); + + VfsTreeNode* DirNode = FindNode(DirName); + + if (!DirNode) + { + // New directory entry (non-virtual) + + DirNode = AddNewChildNode(); + DirNode->Name = std::string(DirName); + DirNode->FileSize = DirectoryMarker; + } + + DirNode->AddVirtualNode(ChildPath, std::move(InDataSource)); + } + else + { + // Add "leaf" node + + VfsTreeNode* LeafNode = nullptr; + + for (auto& Child : GetChildren()) + { + // Already present, just update properties + + if (Child->Name == NodeName) + { + LeafNode = Child.get(); + break; + } + } + + if (!LeafNode) + { + LeafNode = AddNewChildNode(); + } + + LeafNode->Name = std::string(NodeName); + LeafNode->FileSize = DirectoryMarker; + LeafNode->DataSource = std::move(InDataSource); + } +} + +static bool +FileNameLessThan(const std::unique_ptr<VfsTreeNode>& Lhs, const std::unique_ptr<VfsTreeNode>& Rhs) +{ + ExtendableWideStringBuilder<64> Name1, Name2; + + Utf8ToWide(Lhs->GetName(), Name1); + Utf8ToWide(Rhs->GetName(), Name2); + + return PrjFileNameCompare(Name1.c_str(), Name2.c_str()) < 0; +} + +void +VfsTreeNode::NormalizeTree() +{ + // Ensure entries are in the order in which ProjFS expects them. If we + // don't do this then there will be spurious duplicates and things will + // behave strangely + + std::sort(begin(ChildArray), end(ChildArray), FileNameLessThan); + + for (auto& Child : GetChildren()) + { + Child->NormalizeTree(); + } +} + +void +VfsTreeNode::GetPath(StringBuilderBase& Path) const +{ + if (ParentNode) + ParentNode->GetPath(Path); + + if (Path.Size()) + { + Path.Append("\\"); + } + + Path.Append(Name); +} + +const std::string +VfsTreeNode::GetPath() const +{ + ExtendableStringBuilder<64> Path; + GetPath(Path); + return Path.ToString(); +} + +////////////////////////////////////////////////////////////////////////// + +struct VfsTree::Impl +{ + void SetTree(VfsTreeNode&& Tree, VfsTreeDataSource& DataProvider) + { + *m_RootNode = std::move(Tree); + m_RootNode->NormalizeTree(); + m_DataProvider = &DataProvider; + } + VfsTreeNode& GetTree() { return *m_RootNode; } + + void Initialize(VfsTreeDataSource& DataProvider) { m_DataProvider = &DataProvider; } + + void ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length) + { + ZEN_ASSERT(m_DataProvider); + m_DataProvider->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length); + } + + std::unique_ptr<VfsTreeNode> m_RootNode = std::make_unique<VfsTreeNode>(nullptr); + VfsTreeDataSource* m_DataProvider = nullptr; + + const VfsTreeNode* FindNode(std::wstring NodeName) { return m_RootNode->FindNode(NodeName); } +}; + +////////////////////////////////////////////////////////////////////////// + +VfsTree::VfsTree() : m_Impl(new Impl) +{ +} + +VfsTree::~VfsTree() +{ + delete m_Impl; +} + +void +VfsTree::Initialize(VfsTreeDataSource& DataProvider) +{ + m_Impl->Initialize(DataProvider); +} + +void +VfsTree::SetTree(VfsTreeNode&& Tree, VfsTreeDataSource& DataProvider) +{ + m_Impl->SetTree(std::move(Tree), DataProvider); +} + +VfsTreeNode& +VfsTree::GetTree() +{ + return m_Impl->GetTree(); +} + +VfsTreeNode* +VfsTree::FindNode(std::wstring NodeName) +{ + return m_Impl->m_RootNode->FindNode(NodeName); +} + +void +VfsTree::ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length) +{ + m_Impl->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length); +} + +////////////////////////////////////////////////////////////////////////// + +static Guid +ToGuid(const GUID* GuidPtr) +{ + return *reinterpret_cast<const Guid*>(GuidPtr); +} + +static Guid +PathToGuid(std::string_view Path) +{ + const IoHash Hash = IoHash::HashBuffer(Path.data(), Path.size()); + + Guid Id; + + memcpy(&Id.A, &Hash.Hash[0], sizeof Id.A); + memcpy(&Id.B, &Hash.Hash[4], sizeof Id.B); + memcpy(&Id.C, &Hash.Hash[8], sizeof Id.C); + memcpy(&Id.D, &Hash.Hash[12], sizeof Id.D); + + return Id; +} + +void +VfsProvider::EnumerationState::RestartScan() +{ + Iterator = Node->GetChildren().begin(); + EndIterator = Node->GetChildren().end(); +} + +bool +VfsProvider::EnumerationState::MatchesPattern(PCWSTR SearchExpression) +{ + return PrjFileNameMatch(GetCurrentFileName(), SearchExpression); +} + +void +VfsProvider::EnumerationState::MoveToNext() +{ + if (Iterator != EndIterator) + ++Iterator; +} + +const WCHAR* +VfsProvider::EnumerationState::GetCurrentFileName() +{ + CurrentName = zen::Utf8ToWide((*Iterator)->GetName()); + return CurrentName.c_str(); +} + +PRJ_FILE_BASIC_INFO& +VfsProvider::EnumerationState::GetCurrentBasicInfo() +{ + if ((*Iterator)->IsDirectory()) + { + BasicInfo = {.IsDirectory = true, .FileAttributes = FILE_ATTRIBUTE_DIRECTORY}; + } + else + { + BasicInfo = {.IsDirectory = false, .FileSize = INT64((*Iterator)->GetFileSize()), .FileAttributes = FILE_ATTRIBUTE_READONLY}; + } + + return BasicInfo; +} + +////////////////////////////////////////////////////////////////////////// + +VfsProvider::VfsProvider(std::string_view RootPath) : m_RootPath(RootPath) +{ +} + +VfsProvider::~VfsProvider() +{ +} + +void +VfsProvider::Initialize() +{ + std::filesystem::path Root{m_RootPath}; + std::filesystem::path ManifestPath = Root / ".zen_vfs"; + bool HaveManifest = false; + + if (std::filesystem::exists(Root)) + { + if (!std::filesystem::is_directory(Root)) + { + throw std::runtime_error("specified VFS root exists but is not a directory"); + } + + std::error_code Ec; + m_RootPath = WideToUtf8(CanonicalPath(Root, Ec).c_str()); + + if (Ec) + { + throw std::system_error(Ec); + } + + if (std::filesystem::exists(ManifestPath)) + { + FileContents ManifestData = zen::ReadFile(ManifestPath); + CbObject Manifest = LoadCompactBinaryObject(ManifestData.Flatten()); + + m_RootGuid = Manifest["id"].AsUuid(); + + HaveManifest = true; + } + else + { + m_RootGuid = PathToGuid(m_RootPath); + } + } + else + { + zen::CreateDirectories(Root); + m_RootGuid = PathToGuid(m_RootPath); + } + + if (!HaveManifest) + { + CbObjectWriter Cbo; + Cbo << "id" << m_RootGuid; + Cbo << "path" << m_RootPath; + CbObject Manifest = Cbo.Save(); + + zen::WriteFile(ManifestPath, Manifest.GetBuffer().AsIoBuffer()); + } + + m_IsInitialized.test_and_set(); + + // Set up ProjFS root placeholder + + ExtendableWideStringBuilder<256> WideRootPath; + zen::Utf8ToWide(m_RootPath, WideRootPath); + + PRJ_PLACEHOLDER_VERSION_INFO VersionInfo{}; + + if (HRESULT hRes = PrjMarkDirectoryAsPlaceholder(WideRootPath.c_str(), NULL, &VersionInfo, reinterpret_cast<const GUID*>(&m_RootGuid)); + FAILED(hRes)) + { + ThrowSystemException(hRes, fmt::format("PrjMarkDirectoryAsPlaceholder call for '{}' failed", m_RootPath)); + } +} + +void +VfsProvider::Cleanup() +{ + if (m_IsInitialized.test()) + { + zen::CleanDirectoryExceptDotFiles(m_RootPath); + + m_IsInitialized.clear(); + } +} + +void +VfsProvider::AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource) +{ + ZEN_ASSERT(m_IsInitialized.test()); + + if (!m_Tree) + { + m_Tree = new VfsTree; + } + + auto& Tree = m_Tree->GetTree(); + + if (Tree.FindNode(Mountpoint)) + { + throw std::runtime_error(fmt::format("mount point '{}' already exists", Mountpoint)); + } + else + { + Tree.AddVirtualNode(Mountpoint, std::move(DataSource)); + Tree.NormalizeTree(); + } +} + +void +VfsProvider::Run() +{ + const PRJ_CALLBACKS Callbacks = {.StartDirectoryEnumerationCallback = ProjFsProviderInterface::StartDirEnumCallback_C, + .EndDirectoryEnumerationCallback = ProjFsProviderInterface::EndDirEnumCallback_C, + .GetDirectoryEnumerationCallback = ProjFsProviderInterface::GetDirEnumCallback_C, + .GetPlaceholderInfoCallback = ProjFsProviderInterface::GetPlaceholderInfoCallback_C, + .GetFileDataCallback = ProjFsProviderInterface::GetFileDataCallback_C, + /* optional */ .QueryFileNameCallback = ProjFsProviderInterface::QueryFileName_C, + /* optional */ .NotificationCallback = ProjFsProviderInterface::NotificationCallback_C, + /* optional */ .CancelCommandCallback = nullptr /* ProjFsProviderInterface::CancelCommand_C */}; + + PRJ_NOTIFICATION_MAPPING Notifications[] = { + {.NotificationBitMask = PRJ_NOTIFY_FILE_OPENED | PRJ_NOTIFY_PRE_RENAME | PRJ_NOTIFY_PRE_DELETE, .NotificationRoot = L""}}; + + const PRJ_STARTVIRTUALIZING_OPTIONS VirtOptions = {.Flags = PRJ_FLAG_NONE /* PRJ_FLAG_USE_NEGATIVE_PATH_CACHE */, + .PoolThreadCount = 1, + .ConcurrentThreadCount = 1, + .NotificationMappings = Notifications, + .NotificationMappingsCount = ZEN_ARRAY_COUNT(Notifications)}; + + PRJ_NAMESPACE_VIRTUALIZATION_CONTEXT VirtContext{}; + + ExtendableWideStringBuilder<256> WideRootPath; + zen::Utf8ToWide(m_RootPath, WideRootPath); + + if (HRESULT hRes = PrjStartVirtualizing(WideRootPath.c_str(), &Callbacks, this /* InstanceContext */, &VirtOptions, &VirtContext); + FAILED(hRes)) + { + ThrowSystemException(hRes, "PrjStartVirtualizing failed"); + } + + auto StopVirtualizing = MakeGuard([&] { + PrjStopVirtualizing(VirtContext); + m_IsRunning.clear(); + }); + + m_IsRunning.test_and_set(); + + while (!m_StopRunningEvent.Wait(10000 /* ms */)) + { + } +} + +void +VfsProvider::RequestStop() +{ + m_StopRunningEvent.Set(); +} + +////////////////////////////////////////////////////////////////////////// + +HRESULT +VfsProvider::StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) +{ + if (m_IsRunning.test() == false) + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + if (!m_Tree) + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + const std::string Path = WideToUtf8(CallbackData->FilePathName); + const Guid EnumId = ToGuid(EnumerationId); + + VfsTreeNode* EnumNode = &m_Tree->GetTree(); + + if (!Path.empty()) + { + EnumNode = EnumNode->FindNode(CallbackData->FilePathName); + } + + if (!EnumNode) + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + EnumNode->PopulateVirtualNode(); + + auto NodeIt = EnumNode->GetChildren().begin(); + auto EndIt = EnumNode->GetChildren().end(); + + RwLock::ExclusiveLockScope _(m_EnumerationsLock); + m_ActiveEnumerations[EnumId] = EnumerationState{.Path = Path, .Node = EnumNode, .Iterator = NodeIt, .EndIterator = EndIt}; + + return S_OK; +} + +HRESULT +VfsProvider::EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) +{ + ZEN_UNUSED(CallbackData); + + const Guid EnumId = ToGuid(EnumerationId); + + RwLock::ExclusiveLockScope _(m_EnumerationsLock); + m_ActiveEnumerations.erase(EnumId); + + return S_OK; +} + +HRESULT +VfsProvider::GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ const GUID* EnumerationId, + _In_opt_ PCWSTR SearchExpression, + _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) +{ + if (m_IsRunning.test() == false) + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + const Guid EnumId = ToGuid(EnumerationId); + + EnumerationState* State = nullptr; + + { + RwLock::ExclusiveLockScope _(m_EnumerationsLock); + + if (auto EnumIt = m_ActiveEnumerations.find(EnumId); EnumIt == m_ActiveEnumerations.end()) + { + return E_INVALIDARG; + } + else + { + State = &EnumIt->second; + } + } + + if (CallbackData->Flags & PRJ_CB_DATA_FLAG_ENUM_RESTART_SCAN) + { + State->RestartScan(); + } + + bool FilledAtLeastOne = false; + + while (State->IsCurrentValid()) + { + if (State->MatchesPattern(SearchExpression)) + { + HRESULT hRes = PrjFillDirEntryBuffer(State->GetCurrentFileName(), &State->GetCurrentBasicInfo(), DirEntryBufferHandle); + + if (hRes != S_OK) + { + if (FilledAtLeastOne) + { + return S_OK; + } + else + { + return hRes; + } + } + + FilledAtLeastOne = true; + } + + State->MoveToNext(); + } + + return S_OK; +} + +HRESULT +VfsProvider::GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData) +{ + if (m_IsRunning.test() == false) + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + PRJ_PLACEHOLDER_INFO PlaceholderInfo; + memset(&PlaceholderInfo, 0, sizeof PlaceholderInfo); + + std::wstring Path = CallbackData->FilePathName; + + if (const VfsTreeNode* FoundNode = m_Tree->FindNode(Path)) + { + if (FoundNode->IsDirectory()) + { + PlaceholderInfo.FileBasicInfo = {.IsDirectory = true, + .FileAttributes = FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED}; + } + else + { + PlaceholderInfo.FileBasicInfo = {.IsDirectory = false, + .FileSize = static_cast<INT64>(FoundNode->GetFileSize()), + .FileAttributes = FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED}; + } + + // NOTE: for correctness, we should actually provide the canonical casing for the name + // as stored in our backing store rather than simply providing the name which ProjFS + // used for the lookup! + + HRESULT hRes = PrjWritePlaceholderInfo(CallbackData->NamespaceVirtualizationContext, + CallbackData->FilePathName, + &PlaceholderInfo, + sizeof PlaceholderInfo); + + if (FAILED(hRes)) + { + return hRes; + } + + return S_OK; + } + else + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } +} + +HRESULT +VfsProvider::GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) +{ + if (m_IsRunning.test() == false) + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + const VfsTreeNode* FileNode = m_Tree->FindNode(CallbackData->FilePathName); + + if (!FileNode || FileNode->IsDirectory()) + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + void* DataBuffer = PrjAllocateAlignedBuffer(CallbackData->NamespaceVirtualizationContext, Length); + auto _ = MakeGuard([&] { PrjFreeAlignedBuffer(DataBuffer); }); + + if (auto Source = FileNode->GetDataSource()) + { + if (const Oid ChunkId = FileNode->GetChunkId(); ChunkId != Oid::Zero) + { + Source->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length); + } + else + { + Source->ReadNamedData(FileNode->GetName(), DataBuffer, ByteOffset, Length); + } + } + else + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + HRESULT hRes = + PrjWriteFileData(CallbackData->NamespaceVirtualizationContext, &CallbackData->DataStreamId, DataBuffer, ByteOffset, Length); + + if (FAILED(hRes)) + return hRes; + + return S_OK; +} + +HRESULT +VfsProvider::Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ BOOLEAN IsDirectory, + _In_ PRJ_NOTIFICATION NotificationType, + _In_opt_ PCWSTR DestinationFileName, + _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters) +{ + ZEN_UNUSED(CallbackData, IsDirectory, DestinationFileName, NotificationParameters); + + switch (NotificationType) + { + case PRJ_NOTIFICATION_PRE_RENAME: + return HRESULT_FROM_WIN32(ERROR_ACCESS_DENIED); + + case PRJ_NOTIFICATION_PRE_DELETE: + if (m_IsRunning.test()) + { + // don't allow deletes while virtualizing, it doesn't seem + // to make sense as the operation won't be reflected into + // the underlying data source + return HRESULT_FROM_NT(0xC0000121L /* STATUS_CANNOT_DELETE */); + } + else + { + // file deletion is ok during cleanup + return S_OK; + } + + default: + break; + } + + return S_OK; +} + +/** This callback is optional. If the provider does not supply an implementation of this callback, ProjFS will invoke the provider's + directory enumeration callbacks to determine the existence of a file path in the provider's store. + + The provider should use PrjFileNameCompare as the comparison routine when searching its backing store for the specified file. + */ + +HRESULT +VfsProvider::QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData) +{ + if (m_IsRunning.test() == false) + { + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + } + + if (const VfsTreeNode* FileNode = m_Tree->FindNode(CallbackData->FilePathName)) + { + return S_OK; + } + + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); +} + +void +VfsProvider::CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData) +{ + ZEN_UNUSED(CallbackData); +} + +} // namespace zen + +#endif diff --git a/src/zenvfs/vfsprovider.h b/src/zenvfs/vfsprovider.h new file mode 100644 index 000000000..8e6896956 --- /dev/null +++ b/src/zenvfs/vfsprovider.h @@ -0,0 +1,112 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "projfsproviderinterface.h" + +#if ZEN_WITH_VFS + +# include <zencore/compactbinarybuilder.h> +# include <zencore/except.h> +# include <zencore/string.h> +# include <zencore/thread.h> + +ZEN_THIRD_PARTY_INCLUDES_START +# if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +# include <projectedfslib.h> +# endif +ZEN_THIRD_PARTY_INCLUDES_END + +# include <unordered_map> +# include <vector> + +namespace zen { + +class VfsTree +{ +public: + VfsTree(); + ~VfsTree(); + + void Initialize(VfsTreeDataSource& DataProvider); + void SetTree(VfsTreeNode&& TreeRoot, VfsTreeDataSource& DataProvider); + VfsTreeNode& GetTree(); + VfsTreeNode* FindNode(std::wstring NodeName); + void ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length); + +private: + struct Impl; + + Impl* m_Impl; +}; + +////////////////////////////////////////////////////////////////////////// + +struct VfsProvider : public ProjFsProviderInterface +{ + struct EnumerationState + { + std::string Path; + const VfsTreeNode* Node = nullptr; + std::vector<std::unique_ptr<VfsTreeNode>>::const_iterator Iterator; + std::vector<std::unique_ptr<VfsTreeNode>>::const_iterator EndIterator; + + void RestartScan(); + inline bool IsCurrentValid() { return Iterator != EndIterator; } + bool MatchesPattern(PCWSTR SearchExpression); + void MoveToNext(); + const WCHAR* GetCurrentFileName(); + PRJ_FILE_BASIC_INFO& GetCurrentBasicInfo(); + + PRJ_FILE_BASIC_INFO BasicInfo; + std::wstring CurrentName; + }; + + ////////////////////////////////////////////////////////////////////////// + + VfsProvider(std::string_view RootPath); + ~VfsProvider(); + + void Initialize(); + void Cleanup(); + + void AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource); + void RequestStop(); + + // ProjFsProviderInterface implementation + + virtual void Run() override; + + virtual HRESULT StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) override; + virtual HRESULT EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) override; + virtual HRESULT GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ const GUID* EnumerationId, + _In_opt_ PCWSTR SearchExpression, + _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) override; + virtual HRESULT GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData) override; + virtual HRESULT GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) override; + virtual HRESULT Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ BOOLEAN IsDirectory, + _In_ PRJ_NOTIFICATION NotificationType, + _In_opt_ PCWSTR DestinationFileName, + _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters) override; + virtual HRESULT QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData) override; + virtual void CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData) override; + +private: + std::string m_RootPath; + Guid m_RootGuid; + VfsTree* m_Tree = nullptr; + + RwLock m_EnumerationsLock; + std::unordered_map<zen::Guid, EnumerationState> m_ActiveEnumerations; + + Event m_StopRunningEvent; + std::atomic_flag m_IsRunning; + std::atomic_flag m_IsInitialized; +}; + +} // namespace zen + +#endif diff --git a/src/zenvfs/xmake.lua b/src/zenvfs/xmake.lua new file mode 100644 index 000000000..032d02219 --- /dev/null +++ b/src/zenvfs/xmake.lua @@ -0,0 +1,10 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +target('zenvfs') + set_kind("static") + set_group("libs") + add_headerfiles("**.h") + add_files("**.cpp") + add_includedirs("include", {public=true}) + add_deps("zencore") + add_packages("vcpkg::spdlog") diff --git a/src/zenvfs/zenvfs.cpp b/src/zenvfs/zenvfs.cpp new file mode 100644 index 000000000..bec7de473 --- /dev/null +++ b/src/zenvfs/zenvfs.cpp @@ -0,0 +1,13 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenvfs/zenvfs.h> + +namespace zen { + +// temporary dummy function to appease Mac linker +void +ZenVfs() +{ +} + +} // namespace zen |