aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2023-09-20 15:22:03 +0200
committerGitHub <[email protected]>2023-09-20 15:22:03 +0200
commit14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f (patch)
treebf24ac15759385cea339f7e1cf5380f984f5699a
parentchangelog version bump (diff)
downloadzen-14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f.tar.xz
zen-14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f.zip
VFS implementation for local storage service (#396)
currently, only Windows (using Projected File System) is supported
-rw-r--r--CHANGELOG.md5
-rw-r--r--src/zen/cmds/copy.h9
-rw-r--r--src/zen/cmds/up.h4
-rw-r--r--src/zen/cmds/vfs_cmd.cpp108
-rw-r--r--src/zen/cmds/vfs_cmd.h26
-rw-r--r--src/zen/zen.cpp3
-rw-r--r--src/zencore/filesystem.cpp75
-rw-r--r--src/zencore/include/zencore/filesystem.h8
-rw-r--r--src/zenserver/cache/cachedisklayer.cpp25
-rw-r--r--src/zenserver/cache/cachedisklayer.h3
-rw-r--r--src/zenserver/cache/structuredcachestore.cpp18
-rw-r--r--src/zenserver/cache/structuredcachestore.h14
-rw-r--r--src/zenserver/objectstore/objectstore.cpp2
-rw-r--r--src/zenserver/projectstore/projectstore.cpp110
-rw-r--r--src/zenserver/projectstore/projectstore.h17
-rw-r--r--src/zenserver/vfs/vfsimpl.cpp457
-rw-r--r--src/zenserver/vfs/vfsimpl.h100
-rw-r--r--src/zenserver/vfs/vfsservice.cpp217
-rw-r--r--src/zenserver/vfs/vfsservice.h52
-rw-r--r--src/zenserver/xmake.lua3
-rw-r--r--src/zenserver/zenserver.cpp7
-rw-r--r--src/zenvfs/include/zenvfs/projfs.h14
-rw-r--r--src/zenvfs/include/zenvfs/vfs.h89
-rw-r--r--src/zenvfs/include/zenvfs/zenvfs.h9
-rw-r--r--src/zenvfs/projfs.cpp38
-rw-r--r--src/zenvfs/projfsproviderinterface.cpp98
-rw-r--r--src/zenvfs/projfsproviderinterface.h89
-rw-r--r--src/zenvfs/vfs.cpp56
-rw-r--r--src/zenvfs/vfsprovider.cpp794
-rw-r--r--src/zenvfs/vfsprovider.h112
-rw-r--r--src/zenvfs/xmake.lua10
-rw-r--r--src/zenvfs/zenvfs.cpp13
-rw-r--r--xmake.lua1
33 files changed, 2559 insertions, 27 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 868919349..4fe20dc4a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
##
+- Feature: Implemented virtual file system (VFS) support for debugging and introspection purposes
+ - `zen vfs mount <path>` will initialize a virtual file system at the specified mount point. The mount point should ideally not exist already as the server can delete the entirety of it at exit or in other situations. Within the mounted tree you will find directories which allow you to enumerate contents of DDC and the project store
+ - `zen vfs unmount` will stop the VFS
+ - `zen vfs info` can be used to check the status of the VFS
+
## 0.2.23
- Bugfix: Respect result from FinalizeRef in Jupiter oplog upload where it requests missing attachments
- Improvement: Increase timeout when doing import/export of oplogs to jupiter to 30 min per request
diff --git a/src/zen/cmds/copy.h b/src/zen/cmds/copy.h
index 5527ae9b8..549114160 100644
--- a/src/zen/cmds/copy.h
+++ b/src/zen/cmds/copy.h
@@ -18,11 +18,10 @@ public:
virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
private:
- cxxopts::Options m_Options{"copy", "Copy files"};
- std::vector<std::string> m_Positional;
- std::string m_CopySource;
- std::string m_CopyTarget;
- bool m_NoClone = false;
+ cxxopts::Options m_Options{"copy", "Copy files"};
+ std::string m_CopySource;
+ std::string m_CopyTarget;
+ bool m_NoClone = false;
};
} // namespace zen
diff --git a/src/zen/cmds/up.h b/src/zen/cmds/up.h
index 5b5c6a3f8..510cc865e 100644
--- a/src/zen/cmds/up.h
+++ b/src/zen/cmds/up.h
@@ -17,8 +17,8 @@ public:
private:
cxxopts::Options m_Options{"up", "Bring up zen service"};
- uint16_t m_Port;
- int m_OwnerPid;
+ uint16_t m_Port = 0;
+ int m_OwnerPid = 0;
std::string m_ConfigFile;
};
diff --git a/src/zen/cmds/vfs_cmd.cpp b/src/zen/cmds/vfs_cmd.cpp
new file mode 100644
index 000000000..4eb9b3195
--- /dev/null
+++ b/src/zen/cmds/vfs_cmd.cpp
@@ -0,0 +1,108 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "vfs_cmd.h"
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+#include <zenhttp/formatters.h>
+#include <zenhttp/httpclient.h>
+#include <zenutil/zenserverprocess.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+VfsCommand::VfsCommand()
+{
+ m_Options.add_option("", "", "verb", "VFS management verb (mount, unmount)", cxxopts::value(m_Verb), "<verb>");
+ m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<url>");
+ m_Options.add_option("", "", "vfs-path", "Specify VFS mount point path", cxxopts::value(m_MountPath), "<path>");
+
+ m_Options.parse_positional({"verb", "vfs-path"});
+}
+
+VfsCommand::~VfsCommand()
+{
+}
+
+int
+VfsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+ ZEN_UNUSED(GlobalOptions, argc, argv);
+
+ if (!ZenCmdBase::ParseOptions(argc, argv))
+ {
+ return 0;
+ }
+
+ // Validate arguments
+
+ m_HostName = ResolveTargetHostSpec(m_HostName);
+
+ if (m_HostName.empty())
+ throw OptionParseException("unable to resolve server specification");
+
+ HttpClient Http(m_HostName);
+
+ if (m_Verb == "mount"sv)
+ {
+ if (m_MountPath.empty())
+ throw OptionParseException("No source specified");
+
+ CbObjectWriter Cbo;
+ Cbo << "method"
+ << "mount";
+
+ Cbo.BeginObject("params");
+ Cbo << "path" << m_MountPath;
+ Cbo.EndObject();
+
+ if (HttpClient::Response Result = Http.Post("/vfs"sv, Cbo.Save()))
+ {
+ }
+ else
+ {
+ Result.ThrowError("VFS mount request failed"sv);
+
+ return 1;
+ }
+ }
+ else if (m_Verb == "unmount"sv)
+ {
+ CbObjectWriter Cbo;
+ Cbo << "method"
+ << "unmount";
+
+ if (HttpClient::Response Result = Http.Post("/vfs"sv, Cbo.Save()))
+ {
+ }
+ else
+ {
+ Result.ThrowError("VFS unmount request failed"sv);
+
+ return 1;
+ }
+ }
+ else if (m_Verb == "info"sv)
+ {
+ if (HttpClient::Response Result = Http.Get(fmt::format("/vfs/info")))
+ {
+ ExtendableStringBuilder<256> Json;
+ Result.AsObject().ToJson(Json);
+ ZEN_CONSOLE("{}", Json);
+ }
+ else
+ {
+ Result.ThrowError("VFS info fetch failed"sv);
+
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+} // namespace zen
diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h
new file mode 100644
index 000000000..35546c9b6
--- /dev/null
+++ b/src/zen/cmds/vfs_cmd.h
@@ -0,0 +1,26 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+class VfsCommand : public ZenCmdBase
+{
+public:
+ VfsCommand();
+ ~VfsCommand();
+
+ virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+ virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+ cxxopts::Options m_Options{"vfs", "Manage virtual file system"};
+
+ std::string m_Verb;
+ std::string m_HostName;
+ std::string m_MountPath;
+};
+
+} // namespace zen
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 5530b57b6..db67fbae0 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -20,6 +20,7 @@
#include "cmds/top.h"
#include "cmds/up.h"
#include "cmds/version.h"
+#include "cmds/vfs_cmd.h"
#include <zencore/filesystem.h>
#include <zencore/logging.h>
@@ -230,6 +231,7 @@ main(int argc, char** argv)
UpCommand UpCmd;
AttachCommand AttachCmd;
VersionCommand VersionCmd;
+ VfsCommand VfsCmd;
#if ZEN_WITH_TESTS
RunTestsCommand RunTestsCmd;
#endif
@@ -279,6 +281,7 @@ main(int argc, char** argv)
{"up", &UpCmd, "Bring zen server up"},
{"attach", &AttachCmd, "Add a sponsor process to a running zen service"},
{"version", &VersionCmd, "Get zen server version"},
+ {"vfs", &VfsCmd, "Manage virtual file system"},
#if ZEN_WITH_TESTS
{"runtests", &RunTestsCmd, "Run zen tests"},
#endif
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index b1ec14a37..e9f7ba72c 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -92,7 +92,7 @@ CreateDirectories(const wchar_t* Dir)
// behind
static bool
-WipeDirectory(const wchar_t* DirPath)
+WipeDirectory(const wchar_t* DirPath, bool KeepDotFiles)
{
ExtendableWideStringBuilder<128> Pattern;
Pattern.Append(DirPath);
@@ -107,7 +107,7 @@ WipeDirectory(const wchar_t* DirPath)
{
do
{
- bool IsRegular = true;
+ bool AttemptDelete = true;
if (FindData.cFileName[0] == L'.')
{
@@ -115,16 +115,21 @@ WipeDirectory(const wchar_t* DirPath)
{
if (FindData.cFileName[2] == L'\0')
{
- IsRegular = false;
+ AttemptDelete = false;
}
}
else if (FindData.cFileName[1] == L'\0')
{
- IsRegular = false;
+ AttemptDelete = false;
+ }
+
+ if (KeepDotFiles)
+ {
+ AttemptDelete = false;
}
}
- if (IsRegular)
+ if (AttemptDelete)
{
ExtendableWideStringBuilder<128> Path;
Path.Append(DirPath);
@@ -188,7 +193,8 @@ WipeDirectory(const wchar_t* DirPath)
bool
DeleteDirectories(const wchar_t* DirPath)
{
- return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE;
+ const bool KeepDotFiles = false;
+ return WipeDirectory(DirPath, KeepDotFiles) && RemoveDirectoryW(DirPath) == TRUE;
}
bool
@@ -196,7 +202,20 @@ CleanDirectory(const wchar_t* DirPath)
{
if (std::filesystem::exists(DirPath))
{
- return WipeDirectory(DirPath);
+ const bool KeepDotFiles = false;
+
+ return WipeDirectory(DirPath, KeepDotFiles);
+ }
+
+ return CreateDirectories(DirPath);
+}
+
+bool
+CleanDirectory(const wchar_t* DirPath, bool KeepDotFiles)
+{
+ if (std::filesystem::exists(DirPath))
+ {
+ return WipeDirectory(DirPath, KeepDotFiles);
}
return CreateDirectories(DirPath);
@@ -262,6 +281,19 @@ CleanDirectory(const std::filesystem::path& Dir)
#endif
}
+bool
+CleanDirectoryExceptDotFiles(const std::filesystem::path& Dir)
+{
+#if ZEN_PLATFORM_WINDOWS
+ const bool KeepDotFiles = true;
+ return CleanDirectory(Dir.c_str(), KeepDotFiles);
+#else
+ ZEN_UNUSED(Dir);
+
+ ZEN_NOT_IMPLEMENTED();
+#endif
+}
+
//////////////////////////////////////////////////////////////////////////
bool
@@ -950,7 +982,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
if (FAILED(hRes))
{
- ThrowSystemException(hRes, "Failed to open handle to volume root");
+ ThrowSystemException(hRes, fmt::format("Failed to open handle to '{}'", RootDir));
}
while (Continue)
@@ -1063,6 +1095,33 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
closedir(Dir);
#endif // ZEN_PLATFORM_WINDOWS
}
+
+std::filesystem::path
+CanonicalPath(std::filesystem::path InPath, std::error_code& Ec)
+{
+ ZEN_UNUSED(Ec);
+
+#if ZEN_PLATFORM_WINDOWS
+ windows::FileHandle Handle;
+ HRESULT hRes = Handle.Create(InPath.c_str(),
+ GENERIC_READ,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ OPEN_EXISTING,
+ FILE_FLAG_BACKUP_SEMANTICS);
+
+ if (FAILED(hRes))
+ {
+ Ec = MakeErrorCodeFromLastError();
+
+ return {};
+ }
+
+ return PathFromHandle(Handle, Ec);
+#else
+ return InPath;
+#endif
+}
+
std::filesystem::path
PathFromHandle(void* NativeHandle, std::error_code& Ec)
{
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index a78edd16b..075188993 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -30,6 +30,10 @@ ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir);
*/
ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir);
+/** Ensure directory exists and delete contents (if any) before returning
+ */
+ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir);
+
/** Map native file handle to a path
*/
ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle);
@@ -38,6 +42,10 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle);
*/
ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec);
+/** Get canonical path name from a generic path
+ */
+ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec);
+
/** Query file size from native file handle
*/
ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle);
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 7adf07350..9e6f86d79 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -1620,6 +1620,19 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilt
}
void
+ZenCacheDiskLayer::CacheBucket::EnumerateBucketContents(
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const
+{
+ RwLock::SharedLockScope _(m_IndexLock);
+ for (const auto& It : m_Index)
+ {
+ CacheValueDetails::ValueDetails Vd = GetValueDetails(It.first, It.second);
+
+ Fn(It.first, Vd);
+ }
+}
+
+void
ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx)
{
ZEN_TRACE_CPU("Z$::Disk::CollectGarbage");
@@ -2130,6 +2143,18 @@ ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
return {};
}
+void
+ZenCacheDiskLayer::EnumerateBucketContents(std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const
+{
+ RwLock::SharedLockScope _(m_Lock);
+
+ if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
+ {
+ It->second->EnumerateBucketContents(Fn);
+ }
+}
+
CacheValueDetails::NamespaceDetails
ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
{
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index 127e194f1..fc4d8cd6f 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -124,6 +124,8 @@ public:
Info GetInfo() const;
std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
+ void EnumerateBucketContents(std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
@@ -150,6 +152,7 @@ private:
uint64_t EntryCount() const;
CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
+ void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
private:
const uint64_t MaxBlockSize = 1ull << 30;
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index c8384d330..4499b05f7 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -120,6 +120,13 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket)
return AnyDropped;
}
+void
+ZenCacheNamespace::EnumerateBucketContents(std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const
+{
+ m_DiskLayer.EnumerateBucketContents(Bucket, Fn);
+}
+
bool
ZenCacheNamespace::Drop()
{
@@ -502,6 +509,17 @@ ZenCacheStore::GetValueDetails(const std::string_view NamespaceFilter,
return Details;
}
+void
+ZenCacheStore::EnumerateBucketContents(std::string_view Namespace,
+ std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>&& Fn)
+{
+ if (const ZenCacheNamespace* Ns = FindNamespace(Namespace))
+ {
+ Ns->EnumerateBucketContents(Bucket, Fn);
+ }
+}
+
ZenCacheNamespace*
ZenCacheStore::GetNamespace(std::string_view Namespace)
{
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index 8c1f995a4..239efe68f 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -69,10 +69,14 @@ public:
ZenCacheNamespace(GcManager& Gc, const std::filesystem::path& RootDir);
~ZenCacheNamespace();
- bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+
+ bool DropBucket(std::string_view Bucket);
+ void EnumerateBucketContents(std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
+
bool Drop();
- bool DropBucket(std::string_view Bucket);
void Flush();
uint64_t DiskLayerThreshold() const { return m_DiskLayerSizeThreshold; }
@@ -160,6 +164,10 @@ public:
std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket);
std::vector<std::string> GetNamespaces();
+ void EnumerateBucketContents(std::string_view Namespace,
+ std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>&& Fn);
+
private:
const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const;
ZenCacheNamespace* GetNamespace(std::string_view Namespace);
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index e5739418e..3643e8011 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -62,7 +62,7 @@ HttpObjectStoreService::Inititalize()
[this](zen::HttpRouterRequest& Request) {
const std::string BucketName = Request.GetCapture(1);
- StringBuilder<1024> Json;
+ ExtendableStringBuilder<1024> Json;
{
CbObjectWriter Writer;
Writer.BeginArray("distributions");
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 9be600e4e..1ad4403f4 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -691,6 +691,45 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
return {};
}
+std::vector<ProjectStore::Oplog::ChunkInfo>
+ProjectStore::Oplog::GetAllChunksInfo()
+{
+ // First just capture all the chunk ids
+
+ std::vector<ChunkInfo> InfoArray;
+
+ {
+ RwLock::SharedLockScope _(m_OplogLock);
+
+ if (m_Storage)
+ {
+ const size_t NumEntries = m_FileMap.size() + m_ChunkMap.size();
+
+ InfoArray.reserve(NumEntries);
+
+ for (const auto& Kv : m_FileMap)
+ {
+ InfoArray.push_back({.ChunkId = Kv.first});
+ }
+
+ for (const auto& Kv : m_ChunkMap)
+ {
+ InfoArray.push_back({.ChunkId = Kv.first});
+ }
+ }
+ }
+
+ for (ChunkInfo& Info : InfoArray)
+ {
+ if (IoBuffer Chunk = FindChunk(Info.ChunkId))
+ {
+ Info.ChunkSize = Chunk.GetSize();
+ }
+ }
+
+ return InfoArray;
+}
+
void
ProjectStore::Oplog::IterateFileMap(
std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn)
@@ -2056,6 +2095,49 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::strin
}
std::pair<HttpResponseCode, std::string>
+ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload)
+{
+ ZEN_TRACE_CPU("ProjectStore::GetProjectChunks");
+
+ using namespace std::literals;
+
+ Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
+ if (!Project)
+ {
+ return {HttpResponseCode::NotFound, fmt::format("unknown project '{}'", ProjectId)};
+ }
+ Project->TouchProject();
+
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return {HttpResponseCode::NotFound, fmt::format("unknown oplog '{}/{}'", ProjectId, OplogId)};
+ }
+ Project->TouchOplog(OplogId);
+
+ std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfo = FoundLog->GetAllChunksInfo();
+
+ CbObjectWriter Response;
+
+ Response.BeginArray("chunks"sv);
+ for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
+ {
+ Response << Info.ChunkId;
+ }
+ Response.EndArray();
+
+ Response.BeginArray("sizes"sv);
+ for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
+ {
+ Response << Info.ChunkSize;
+ }
+ Response.EndArray();
+
+ OutPayload = Response.Save();
+ return {HttpResponseCode::OK, {}};
+}
+
+std::pair<HttpResponseCode, std::string>
ProjectStore::GetChunkInfo(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view ChunkId,
@@ -2120,6 +2202,25 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
ZenContentType AcceptType,
IoBuffer& OutChunk)
{
+ if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
+ {
+ return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)};
+ }
+
+ const Oid Obj = Oid::FromHexString(ChunkId);
+
+ return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk);
+}
+
+std::pair<HttpResponseCode, std::string>
+ProjectStore::GetChunkRange(const std::string_view ProjectId,
+ const std::string_view OplogId,
+ Oid ChunkId,
+ uint64_t Offset,
+ uint64_t Size,
+ ZenContentType AcceptType,
+ IoBuffer& OutChunk)
+{
bool IsOffset = Offset != 0 || Size != ~(0ull);
Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
@@ -2136,14 +2237,7 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
}
Project->TouchOplog(OplogId);
- if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
- {
- return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)};
- }
-
- const Oid Obj = Oid::FromHexString(ChunkId);
-
- IoBuffer Chunk = FoundLog->FindChunk(Obj);
+ IoBuffer Chunk = FoundLog->FindChunk(ChunkId);
if (!Chunk)
{
return {HttpResponseCode::NotFound, {}};
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index aa84d04ca..9c0f8790d 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -85,6 +85,13 @@ public:
void Write();
void Update(const std::filesystem::path& MarkerPath);
+ struct ChunkInfo
+ {
+ Oid ChunkId;
+ uint64_t ChunkSize;
+ };
+
+ std::vector<ChunkInfo> GetAllChunksInfo();
void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
void IterateOplog(std::function<void(CbObject)>&& Fn);
void IterateOplogWithKey(std::function<void(int, const Oid&, CbObject)>&& Fn);
@@ -287,12 +294,22 @@ public:
const std::string_view OplogId,
bool FilterClient,
CbObject& OutPayload);
+ std::pair<HttpResponseCode, std::string> GetProjectChunks(const std::string_view ProjectId,
+ const std::string_view OplogId,
+ CbObject& OutPayload);
std::pair<HttpResponseCode, std::string> GetChunkInfo(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view ChunkId,
CbObject& OutPayload);
std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId,
const std::string_view OplogId,
+ const Oid ChunkId,
+ uint64_t Offset,
+ uint64_t Size,
+ ZenContentType AcceptType,
+ IoBuffer& OutChunk);
+ std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId,
+ const std::string_view OplogId,
const std::string_view ChunkId,
uint64_t Offset,
uint64_t Size,
diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp
new file mode 100644
index 000000000..f74000900
--- /dev/null
+++ b/src/zenserver/vfs/vfsimpl.cpp
@@ -0,0 +1,457 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "vfsimpl.h"
+#include "vfsservice.h"
+
+#include "cache/structuredcachestore.h"
+#include "projectstore/projectstore.h"
+
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zenvfs/projfs.h>
+#include <zenvfs/vfs.h>
+
+#include <memory>
+
+#if ZEN_WITH_VFS
+
+namespace zen {
+
+using namespace std::literals;
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsOplogDataSource::VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore)
+: m_ProjectId(ProjectId)
+, m_OplogId(OplogId)
+, m_ProjectStore(std::move(InProjectStore))
+{
+}
+
+void
+VfsOplogDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount);
+}
+
+void
+VfsOplogDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ IoBuffer ChunkBuffer;
+ auto Result =
+ m_ProjectStore->GetChunkRange(m_ProjectId, m_OplogId, ChunkId, 0, ~0ull, ZenContentType::kCompressedBinary, /* out */ ChunkBuffer);
+
+ if (Result.first == HttpResponseCode::OK)
+ {
+ const uint8_t* SourceBuffer = reinterpret_cast<const uint8_t*>(ChunkBuffer.GetData());
+ uint64_t AvailableBufferBytes = ChunkBuffer.GetSize();
+
+ ZEN_ASSERT(AvailableBufferBytes >= ByteOffset);
+ AvailableBufferBytes -= ByteOffset;
+ SourceBuffer += ByteOffset;
+
+ ZEN_ASSERT(AvailableBufferBytes >= ByteCount);
+ memcpy(Buffer, SourceBuffer, ByteCount);
+ }
+}
+
+void
+VfsOplogDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
+{
+ // This should never be called
+ ZEN_UNUSED(NodePath, DirNode);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsCacheDataSource::VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore)
+: m_NamespaceId(NamespaceId)
+, m_BucketId(BucketId)
+, m_CacheStore(std::move(InCacheStore))
+{
+}
+
+VfsCacheDataSource::~VfsCacheDataSource()
+{
+}
+
+void
+VfsCacheDataSource::ReadNamedData(std::string_view Name, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ if (auto DotIndex = Name.find_first_of('.'); DotIndex != std::string_view::npos)
+ {
+ Name = Name.substr(0, DotIndex);
+ }
+
+ IoHash HashKey = IoHash::FromHexString(Name);
+
+ CacheRequestContext CacheContext{};
+
+ ZenCacheValue Value;
+ if (m_CacheStore->Get(CacheContext, m_NamespaceId, m_BucketId, HashKey, /* out */ Value))
+ {
+ // TODO bounds check!
+ auto DataPtr = reinterpret_cast<const uint8_t*>(Value.Value.GetData()) + ByteOffset;
+
+ memcpy(Buffer, DataPtr, ByteCount);
+
+ return;
+ }
+}
+
+void
+VfsCacheDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount);
+}
+
+void
+VfsCacheDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
+{
+ ZEN_UNUSED(NodePath, DirNode);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsService::Impl::Impl()
+{
+}
+
+VfsService::Impl::~Impl()
+{
+ Unmount();
+}
+
+void
+VfsService::Impl::Mount(std::string_view MountPoint)
+{
+ ZEN_INFO("VFS mount requested at '{}'", MountPoint);
+
+# if ZEN_PLATFORM_WINDOWS
+ if (!IsProjFsAvailable())
+ {
+ throw std::runtime_error("Projected File System component not available");
+ }
+# endif
+
+ if (!m_MountpointPath.empty())
+ {
+ throw std::runtime_error("VFS already mounted");
+ }
+
+ m_MountpointPath = MountPoint;
+
+ RefreshVfs();
+}
+
+void
+VfsService::Impl::Unmount()
+{
+ if (m_MountpointPath.empty())
+ {
+ return;
+ }
+
+ ZEN_INFO("unmounting VFS from '{}'", m_MountpointPath);
+
+ m_MountpointPath.clear();
+
+ RefreshVfs();
+}
+
+void
+VfsService::Impl::AddService(Ref<ProjectStore>&& Ps)
+{
+ m_ProjectStore = std::move(Ps);
+
+ RefreshVfs();
+}
+
+void
+VfsService::Impl::AddService(Ref<ZenCacheStore>&& Z$)
+{
+ m_ZenCacheStore = std::move(Z$);
+
+ RefreshVfs();
+}
+
+void
+VfsService::Impl::RefreshVfs()
+{
+ if (m_VfsHost && m_MountpointPath.empty())
+ {
+ m_VfsHost->RequestStop();
+ m_VfsThread.join();
+ m_VfsHost.reset();
+ m_VfsThreadRunning.Reset();
+ m_VfsDataSource = nullptr;
+
+ return;
+ }
+
+ if (!m_VfsHost && !m_MountpointPath.empty())
+ {
+ m_VfsThread = std::thread(&VfsService::Impl::VfsThread, this);
+ m_VfsThreadRunning.Wait();
+
+ // At this stage, m_VfsHost should be initialized
+
+ ZEN_ASSERT(m_VfsHost);
+ }
+
+ if (m_ProjectStore && m_VfsHost)
+ {
+ if (!m_VfsDataSource)
+ {
+ m_VfsDataSource = new VfsServiceDataSource(this);
+ }
+
+ m_VfsHost->AddMount("projects"sv, m_VfsDataSource);
+ }
+
+ if (m_ZenCacheStore && m_VfsHost)
+ {
+ if (!m_VfsDataSource)
+ {
+ m_VfsDataSource = new VfsServiceDataSource(this);
+ }
+
+ m_VfsHost->AddMount("ddc_cache"sv, m_VfsDataSource);
+ }
+}
+
+void
+VfsService::Impl::VfsThread()
+{
+ SetCurrentThreadName("VFS");
+
+ ZEN_INFO("VFS service thread now RUNNING");
+
+ try
+ {
+ m_VfsHost = std::make_unique<VfsHost>(m_MountpointPath);
+ m_VfsHost->Initialize();
+
+ m_VfsThreadRunning.Set();
+ m_VfsHost->Run();
+ }
+ catch (std::exception& Ex)
+ {
+ ZEN_WARN("exception caught in VFS thread: {}", Ex.what());
+
+ m_VfsThreadException = std::current_exception();
+ }
+
+ if (m_VfsHost)
+ {
+ m_VfsHost->Cleanup();
+ }
+
+ ZEN_INFO("VFS service thread now EXITING");
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+Ref<VfsOplogDataSource>
+VfsServiceDataSource::GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId)
+{
+ ExtendableStringBuilder<256> Key;
+ Key << ProjectId << "." << OplogId;
+ std::string StdKey{Key};
+
+ RwLock::ExclusiveLockScope _(m_Lock);
+
+ if (auto It = m_OplogSourceMap.find(StdKey); It == m_OplogSourceMap.end())
+ {
+ Ref<VfsOplogDataSource> NewSource{new VfsOplogDataSource(ProjectId, OplogId, m_VfsImpl->m_ProjectStore)};
+ m_OplogSourceMap[StdKey] = NewSource;
+ return NewSource;
+ }
+ else
+ {
+ return It->second;
+ }
+}
+
+Ref<VfsCacheDataSource>
+VfsServiceDataSource::GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId)
+{
+ ExtendableStringBuilder<256> Key;
+ Key << NamespaceId << "." << BucketId;
+ std::string StdKey{Key};
+
+ RwLock::ExclusiveLockScope _(m_Lock);
+
+ if (auto It = m_CacheSourceMap.find(StdKey); It == m_CacheSourceMap.end())
+ {
+ Ref<VfsCacheDataSource> NewSource{new VfsCacheDataSource(NamespaceId, BucketId, m_VfsImpl->m_ZenCacheStore)};
+ m_CacheSourceMap[StdKey] = NewSource;
+ return NewSource;
+ }
+ else
+ {
+ return It->second;
+ }
+}
+
+void
+VfsServiceDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount);
+}
+
+void
+VfsServiceDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount);
+}
+
+void
+VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
+{
+ if (NodePath == "projects"sv)
+ {
+ // Project enumeration
+
+ m_VfsImpl->m_ProjectStore->DiscoverProjects();
+
+ m_VfsImpl->m_ProjectStore->IterateProjects(
+ [&](ProjectStore::Project& Project) { DirNode.AddVirtualNode(Project.Identifier, m_VfsImpl->m_VfsDataSource); });
+ }
+ else if (NodePath.starts_with("projects\\"sv))
+ {
+ std::string_view ProjectId{NodePath};
+ ProjectId = ProjectId.substr(9); // Skip "projects\"
+
+ if (std::string_view::size_type SlashOffset = ProjectId.find_first_of('\\'); SlashOffset == std::string_view::npos)
+ {
+ Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId);
+
+ if (!Project)
+ {
+ // No such project found?
+
+ return;
+ }
+
+ // Oplog enumeration
+
+ std::vector<std::string> Oplogs = Project->ScanForOplogs();
+
+ for (auto& Oplog : Oplogs)
+ {
+ DirNode.AddVirtualNode(Oplog, m_VfsImpl->m_VfsDataSource);
+ }
+ }
+ else
+ {
+ std::string_view OplogId = ProjectId.substr(SlashOffset + 1);
+ ProjectId = ProjectId.substr(0, SlashOffset);
+
+ Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId);
+
+ if (!Project)
+ {
+ // No such project found?
+
+ return;
+ }
+
+ // Oplog contents enumeration
+
+ if (ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId))
+ {
+ Ref<VfsOplogDataSource> DataSource = GetOplogDataSource(ProjectId, OplogId);
+
+ // Get metadata for all chunks
+ std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfos = Oplog->GetAllChunksInfo();
+
+ std::unordered_map<zen::Oid, uint64_t> ChunkSizes;
+
+ for (const auto& Ci : ChunkInfos)
+ {
+ ChunkSizes[Ci.ChunkId] = Ci.ChunkSize;
+ }
+
+ auto EmitFilesForDataArray = [&](zen::CbArrayView DataArray) {
+ for (auto DataIter : DataArray)
+ {
+ if (zen::CbObjectView Data = DataIter.AsObjectView())
+ {
+ std::string_view FileName = Data["filename"sv].AsString();
+ zen::Oid ChunkId = Data["id"sv].AsObjectId();
+
+ if (auto FindIt = ChunkSizes.find(ChunkId); FindIt != ChunkSizes.end())
+ {
+ DirNode.AddFileNode(FileName, FindIt->second /* file size */, ChunkId, DataSource);
+ }
+ else
+ {
+ ZEN_WARN("no chunk metadata found for chunk {} (file: '{}')", ChunkId, FileName);
+ }
+ }
+ }
+ };
+
+ Oplog->IterateOplog([&](CbObject Op) {
+ EmitFilesForDataArray(Op["packagedata"sv].AsArrayView());
+ EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView());
+ });
+
+ DirNode.AddFileNode("stats.json", 42, Oid::Zero);
+ }
+ }
+ }
+ else if (NodePath == "ddc_cache"sv)
+ {
+ // Namespace enumeration
+
+ std::vector<std::string> Namespaces = m_VfsImpl->m_ZenCacheStore->GetNamespaces();
+
+ for (auto& Namespace : Namespaces)
+ {
+ DirNode.AddVirtualNode(Namespace, m_VfsImpl->m_VfsDataSource);
+ }
+ }
+ else if (NodePath.starts_with("ddc_cache\\"sv))
+ {
+ std::string_view NamespaceId{NodePath};
+ NamespaceId = NamespaceId.substr(10); // Skip "ddc_cache\"
+
+ auto& Cache = m_VfsImpl->m_ZenCacheStore;
+
+ if (std::string_view::size_type SlashOffset = NamespaceId.find_first_of('\\'); SlashOffset == std::string_view::npos)
+ {
+ // Bucket enumeration
+
+ if (auto NsInfo = Cache->GetNamespaceInfo(NamespaceId))
+ {
+ for (auto& BucketName : NsInfo->BucketNames)
+ {
+ DirNode.AddVirtualNode(BucketName, m_VfsImpl->m_VfsDataSource);
+ }
+ }
+ }
+ else
+ {
+ // Bucket contents enumeration
+
+ std::string_view BucketId = NamespaceId.substr(SlashOffset + 1);
+ NamespaceId = NamespaceId.substr(0, SlashOffset);
+
+ Ref<VfsCacheDataSource> DataSource = GetCacheDataSource(NamespaceId, BucketId);
+
+ auto Enumerator = [&](const IoHash& Key, const CacheValueDetails::ValueDetails& Details) {
+ ExtendableStringBuilder<64> KeyString;
+ Key.ToHexString(KeyString);
+ KeyString.Append(".udd");
+ DirNode.AddFileNode(KeyString, Details.Size, Oid::Zero, DataSource);
+ };
+
+ Cache->EnumerateBucketContents(NamespaceId, BucketId, Enumerator);
+ }
+ }
+}
+
+} // namespace zen
+#endif
diff --git a/src/zenserver/vfs/vfsimpl.h b/src/zenserver/vfs/vfsimpl.h
new file mode 100644
index 000000000..9e4fdfe99
--- /dev/null
+++ b/src/zenserver/vfs/vfsimpl.h
@@ -0,0 +1,100 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "vfsservice.h"
+
+#include "projectstore/projectstore.h"
+
+#include <zencore/logging.h>
+#include <zenvfs/vfs.h>
+
+#if ZEN_WITH_VFS
+
+# include <memory>
+
+namespace zen {
+
+struct VfsOplogDataSource : public VfsTreeDataSource
+{
+ VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore);
+
+ virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
+
+private:
+ std::string m_ProjectId;
+ std::string m_OplogId;
+ Ref<ProjectStore> m_ProjectStore;
+};
+
+struct VfsCacheDataSource : public VfsTreeDataSource
+{
+ VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore);
+ ~VfsCacheDataSource();
+
+ virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
+
+private:
+ std::string m_NamespaceId;
+ std::string m_BucketId;
+ Ref<ZenCacheStore> m_CacheStore;
+};
+
+struct VfsServiceDataSource : public VfsTreeDataSource
+{
+ VfsServiceDataSource(VfsService::Impl* VfsImpl) : m_VfsImpl(VfsImpl) {}
+
+ virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
+
+private:
+ VfsService::Impl* m_VfsImpl = nullptr;
+
+ RwLock m_Lock;
+ std::unordered_map<std::string, Ref<VfsOplogDataSource>> m_OplogSourceMap;
+ std::unordered_map<std::string, Ref<VfsCacheDataSource>> m_CacheSourceMap;
+
+ Ref<VfsOplogDataSource> GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId);
+ Ref<VfsCacheDataSource> GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId);
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct VfsService::Impl
+{
+ Impl();
+ ~Impl();
+
+ void Mount(std::string_view MountPoint);
+ void Unmount();
+ void AddService(Ref<ProjectStore>&&);
+ void AddService(Ref<ZenCacheStore>&&);
+
+ inline std::string GetMountpointPath() { return m_MountpointPath; }
+ inline bool IsVfsRunning() const { return !!m_VfsHost.get(); }
+
+private:
+ Ref<ProjectStore> m_ProjectStore;
+ Ref<ZenCacheStore> m_ZenCacheStore;
+ Ref<VfsServiceDataSource> m_VfsDataSource;
+ std::string m_MountpointPath;
+
+ std::unique_ptr<VfsHost> m_VfsHost;
+ std::thread m_VfsThread;
+ Event m_VfsThreadRunning;
+ std::exception_ptr m_VfsThreadException;
+
+ void RefreshVfs();
+ void VfsThread();
+
+ friend struct VfsServiceDataSource;
+};
+
+} // namespace zen
+
+#endif
diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp
new file mode 100644
index 000000000..c53682d93
--- /dev/null
+++ b/src/zenserver/vfs/vfsservice.cpp
@@ -0,0 +1,217 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "vfsservice.h"
+#include "vfsimpl.h"
+
+#include <zencore/compactbinarybuilder.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+#if ZEN_WITH_VFS
+
+//////////////////////////////////////////////////////////////////////////
+
+bool
+GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb)
+{
+ IoBuffer Payload = HttpReq.ReadPayload();
+ HttpContentType PayloadContentType = HttpReq.RequestContentType();
+
+ switch (PayloadContentType)
+ {
+ case HttpContentType::kJSON:
+ case HttpContentType::kUnknownContentType:
+ case HttpContentType::kText:
+ {
+ std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize());
+ Cb = LoadCompactBinaryFromJson(JsonText).AsObject();
+ if (!Cb)
+ {
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Content format not supported, expected JSON format");
+ return false;
+ }
+ }
+ break;
+ case HttpContentType::kCbObject:
+ Cb = LoadCompactBinaryObject(Payload);
+ if (!Cb)
+ {
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Content format not supported, expected compact binary format");
+ return false;
+ }
+ break;
+ default:
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+ return false;
+ }
+
+ return true;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// to test:
+//
+// echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:1337/vfs --data-binary @-
+// echo {"method": "unmount"} | curl.exe http://localhost:1337/vfs --data-binary @-
+
+VfsService::VfsService()
+{
+ m_Impl = new Impl;
+
+ m_Router.RegisterRoute(
+ "info",
+ [&](HttpRouterRequest& Request) {
+ CbObjectWriter Cbo;
+ Cbo << "running" << m_Impl->IsVfsRunning();
+ Cbo << "rootpath" << m_Impl->GetMountpointPath();
+
+ Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ },
+ HttpVerb::kGet | HttpVerb::kHead);
+
+ m_Router.RegisterRoute(
+ "",
+ [&](HttpRouterRequest& Req) {
+ CbObject Payload;
+
+ if (!GetContentAsCbObject(Req.ServerRequest(), Payload))
+ return;
+
+ std::string_view RpcName = Payload["method"sv].AsString();
+
+ if (RpcName == "mount"sv)
+ {
+ CbObjectView Params = Payload["params"sv].AsObjectView();
+ std::string_view Mountpath = Params["path"sv].AsString();
+
+ if (Mountpath.empty())
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "no path specified");
+ }
+
+ if (m_Impl->IsVfsRunning())
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS already mounted");
+ }
+
+ try
+ {
+ m_Impl->Mount(Mountpath);
+ }
+ catch (std::exception& Ex)
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what());
+ }
+
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
+ }
+ else if (RpcName == "unmount"sv)
+ {
+ if (!m_Impl->IsVfsRunning())
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS not active");
+ }
+
+ try
+ {
+ m_Impl->Unmount();
+ }
+ catch (std::exception& Ex)
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what());
+ }
+
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
+ }
+ else
+ {
+ Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "unknown RPC"sv);
+ }
+ },
+ HttpVerb::kPost);
+}
+
+VfsService::~VfsService()
+{
+ delete m_Impl;
+}
+
+void
+VfsService::Mount(std::string_view MountPoint)
+{
+ m_Impl->Mount(MountPoint);
+}
+
+void
+VfsService::Unmount()
+{
+ m_Impl->Unmount();
+}
+
+void
+VfsService::AddService(Ref<ProjectStore>&& Ps)
+{
+ m_Impl->AddService(std::move(Ps));
+}
+
+void
+VfsService::AddService(Ref<ZenCacheStore>&& Z$)
+{
+ m_Impl->AddService(std::move(Z$));
+}
+
+#else
+
+VfsService::VfsService()
+{
+}
+
+VfsService::~VfsService()
+{
+}
+
+void
+VfsService::Mount(std::string_view MountPoint)
+{
+ ZEN_UNUSED(MountPoint);
+}
+
+void
+VfsService::Unmount()
+{
+}
+
+void
+VfsService::AddService(Ref<ProjectStore>&& Ps)
+{
+ ZEN_UNUSED(Ps);
+}
+
+void
+VfsService::AddService(Ref<ZenCacheStore>&& Z$)
+{
+ ZEN_UNUSED(Z$);
+}
+
+#endif
+
+const char*
+VfsService::BaseUri() const
+{
+ return "/vfs/";
+}
+
+void
+VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest)
+{
+ m_Router.HandleRequest(HttpServiceRequest);
+}
+
+} // namespace zen
diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h
new file mode 100644
index 000000000..9510cfcda
--- /dev/null
+++ b/src/zenserver/vfs/vfsservice.h
@@ -0,0 +1,52 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/refcount.h>
+#include <zenhttp/httpserver.h>
+#include <zenvfs/vfs.h>
+
+#include <memory>
+
+namespace zen {
+
+class ProjectStore;
+class ZenCacheStore;
+
+/** Virtual File System service
+
+ Implements support for exposing data via a virtual file system interface. Currently
+ this is primarily used to surface various data stored in the local storage service
+ to users for debugging and exploration purposes.
+
+ Currently, it surfaces information from the structured cache service and from the
+ project store.
+
+ */
+
+class VfsService : public HttpService
+{
+public:
+ VfsService();
+ ~VfsService();
+
+ void Mount(std::string_view MountPoint);
+ void Unmount();
+
+ void AddService(Ref<ProjectStore>&&);
+ void AddService(Ref<ZenCacheStore>&&);
+
+protected:
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override;
+
+private:
+ struct Impl;
+ Impl* m_Impl = nullptr;
+
+ HttpRequestRouter m_Router;
+
+ friend struct VfsServiceDataSource;
+};
+
+} // namespace zen
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index a2d02baae..66bfe4ada 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -3,6 +3,7 @@
target("zenserver")
set_kind("binary")
add_deps("zencore", "zenhttp", "zenstore", "zenutil")
+ add_deps("zenvfs")
add_headerfiles("**.h")
add_files("**.cpp")
add_files("zenserver.cpp", {unity_ignored = true })
@@ -19,6 +20,8 @@ target("zenserver")
add_ldflags("/LTCG")
add_files("zenserver.rc")
add_cxxflags("/bigobj")
+ add_links("delayimp", "projectedfslib")
+ add_ldflags("/delayload:ProjectedFSLib.dll")
else
remove_files("windows/**")
end
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 988f72273..1f37e336f 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -125,6 +125,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
#include "projectstore/httpprojectstore.h"
#include "projectstore/projectstore.h"
#include "upstream/upstream.h"
+#include "vfs/vfsservice.h"
#define ZEN_APP_NAME "Zen store"
@@ -419,6 +420,11 @@ public:
m_Http->RegisterService(*m_ObjStoreService);
}
+ m_VfsService = std::make_unique<VfsService>();
+ m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore));
+ m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore));
+ m_Http->RegisterService(*m_VfsService);
+
ZEN_INFO("initializing GC, enabled '{}', interval {}s", ServerOptions.GcConfig.Enabled, ServerOptions.GcConfig.IntervalSeconds);
zen::GcSchedulerConfig GcConfig{
.RootDirectory = m_DataRoot / "gc",
@@ -750,6 +756,7 @@ private:
#endif // ZEN_WITH_COMPUTE_SERVICES
std::unique_ptr<zen::HttpFrontendService> m_FrontendService;
std::unique_ptr<zen::HttpObjectStoreService> m_ObjStoreService;
+ std::unique_ptr<zen::VfsService> m_VfsService;
std::unique_ptr<JobQueue> m_JobQueue;
std::unique_ptr<zen::HttpAdminService> m_AdminService;
diff --git a/src/zenvfs/include/zenvfs/projfs.h b/src/zenvfs/include/zenvfs/projfs.h
new file mode 100644
index 000000000..8a89c2835
--- /dev/null
+++ b/src/zenvfs/include/zenvfs/projfs.h
@@ -0,0 +1,14 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenvfs.h"
+
+#if ZEN_WITH_VFS
+
+namespace zen {
+
+bool IsProjFsAvailable();
+
+} // namespace zen
+#endif
diff --git a/src/zenvfs/include/zenvfs/vfs.h b/src/zenvfs/include/zenvfs/vfs.h
new file mode 100644
index 000000000..412970860
--- /dev/null
+++ b/src/zenvfs/include/zenvfs/vfs.h
@@ -0,0 +1,89 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenvfs.h"
+
+#if ZEN_WITH_VFS
+
+# include <zencore/zencore.h>
+# include <zencore/refcount.h>
+# include <zencore/uid.h>
+
+# include <string_view>
+# include <memory>
+# include <vector>
+
+namespace zen {
+
+struct VfsProvider;
+
+//////////////////////////////////////////////////////////////////////////
+
+struct VfsTreeNode;
+
+struct VfsTreeDataSource : public RefCounted
+{
+ virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) = 0;
+ virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) = 0;
+ virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) = 0;
+};
+
+struct VfsTreeNode
+{
+ inline VfsTreeNode(VfsTreeNode* InParent) : ParentNode(InParent) {}
+
+ const std::string_view GetName() const { return Name; }
+ uint64_t GetFileSize() const { return FileSize; }
+ VfsTreeDataSource* GetDataSource() const { return DataSource.Get(); }
+ const Oid& GetChunkId() const { return ChunkId; }
+
+ const std::string GetPath() const;
+
+ VfsTreeNode* FindNode(std::wstring_view NodeName);
+ VfsTreeNode* FindNode(std::string_view NodeName);
+
+ void AddVirtualNode(std::string_view NodeName, Ref<VfsTreeDataSource>&& DataSource);
+ void AddFileNode(std::string_view NodeName, uint64_t InFileSize, Oid InChunkId, Ref<VfsTreeDataSource> DataSource = {});
+ void NormalizeTree();
+ bool IsDirectory() const;
+ void PopulateVirtualNode();
+
+ const std::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() const { return ChildArray; }
+ std::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() { return ChildArray; }
+
+private:
+ std::string Name;
+ uint64_t FileSize = 0;
+ Oid ChunkId = Oid::Zero;
+ Ref<VfsTreeDataSource> DataSource;
+
+ inline static const uint64_t DirectoryMarker = ~uint64_t(0);
+ std::vector<std::unique_ptr<VfsTreeNode>> ChildArray;
+ VfsTreeNode* ParentNode = nullptr;
+
+ VfsTreeNode* AddNewChildNode() { return ChildArray.emplace_back(std::make_unique<VfsTreeNode>(this)).get(); }
+
+ void GetPath(StringBuilderBase& Path) const;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+class VfsHost
+{
+public:
+ VfsHost(std::string_view VfsRootPath);
+ ~VfsHost();
+
+ void Initialize();
+ void AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource);
+ void Run();
+ void RequestStop();
+ void Cleanup();
+
+private:
+ VfsProvider* m_Provider = nullptr;
+};
+
+} // namespace zen
+#endif
diff --git a/src/zenvfs/include/zenvfs/zenvfs.h b/src/zenvfs/include/zenvfs/zenvfs.h
new file mode 100644
index 000000000..c70368a1d
--- /dev/null
+++ b/src/zenvfs/include/zenvfs/zenvfs.h
@@ -0,0 +1,9 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#ifndef ZEN_WITH_VFS
+# define ZEN_WITH_VFS ZEN_PLATFORM_WINDOWS
+#endif
diff --git a/src/zenvfs/projfs.cpp b/src/zenvfs/projfs.cpp
new file mode 100644
index 000000000..5d74337b4
--- /dev/null
+++ b/src/zenvfs/projfs.cpp
@@ -0,0 +1,38 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenvfs/projfs.h"
+
+#if ZEN_WITH_VFS
+# if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+
+namespace zen {
+
+bool
+IsProjFsAvailable()
+{
+ HMODULE hMod = LoadLibraryA("projectedfslib.dll");
+
+ if (hMod)
+ {
+ FreeLibrary(hMod);
+
+ return true;
+ }
+
+ return false;
+}
+
+} // namespace zen
+# else
+namespace zen {
+
+bool
+IsProjFsAvailable()
+{
+ return false;
+}
+
+} // namespace zen
+# endif
+#endif
diff --git a/src/zenvfs/projfsproviderinterface.cpp b/src/zenvfs/projfsproviderinterface.cpp
new file mode 100644
index 000000000..ca4b91382
--- /dev/null
+++ b/src/zenvfs/projfsproviderinterface.cpp
@@ -0,0 +1,98 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "projfsproviderinterface.h"
+
+#if ZEN_WITH_VFS
+
+namespace zen {
+
+HRESULT
+ProjFsProviderInterface::Notify(const PRJ_CALLBACK_DATA* CallbackData,
+ BOOLEAN IsDirectory,
+ PRJ_NOTIFICATION NotificationType,
+ PCWSTR DestinationFileName,
+ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters)
+{
+ ZEN_UNUSED(CallbackData, IsDirectory, NotificationType, DestinationFileName, NotificationParameters);
+ return S_OK;
+}
+
+HRESULT
+ProjFsProviderInterface::QueryFileName(const PRJ_CALLBACK_DATA* CallbackData)
+{
+ ZEN_UNUSED(CallbackData);
+ return S_OK;
+}
+
+void
+ProjFsProviderInterface::CancelCommand(const PRJ_CALLBACK_DATA* CallbackData)
+{
+ ZEN_UNUSED(CallbackData);
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Forwarding functions for all ProjFS callbacks - these simply call the
+// corresponding member function on the active provider
+//
+
+HRESULT
+ProjFsProviderInterface::StartDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->StartDirEnum(CallbackData, EnumerationId);
+}
+
+HRESULT
+ProjFsProviderInterface::EndDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->EndDirEnum(CallbackData, EnumerationId);
+}
+
+HRESULT
+ProjFsProviderInterface::GetDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)
+ ->GetDirEnum(CallbackData, EnumerationId, SearchExpression, DirEntryBufferHandle);
+}
+
+HRESULT
+ProjFsProviderInterface::GetPlaceholderInfoCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->GetPlaceholderInfo(CallbackData);
+}
+
+HRESULT
+ProjFsProviderInterface::GetFileDataCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->GetFileData(CallbackData, ByteOffset, Length);
+}
+
+HRESULT
+ProjFsProviderInterface::NotificationCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)
+ ->Notify(CallbackData, IsDirectory, NotificationType, DestinationFileName, NotificationParameters);
+}
+
+HRESULT
+ProjFsProviderInterface::QueryFileName_C(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->QueryFileName(CallbackData);
+}
+
+void
+ProjFsProviderInterface::CancelCommand_C(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->CancelCommand(CallbackData);
+}
+
+} // namespace zen
+
+#endif
diff --git a/src/zenvfs/projfsproviderinterface.h b/src/zenvfs/projfsproviderinterface.h
new file mode 100644
index 000000000..33ea4b177
--- /dev/null
+++ b/src/zenvfs/projfsproviderinterface.h
@@ -0,0 +1,89 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenvfs/vfs.h>
+#include <zenvfs/zenvfs.h>
+
+#if ZEN_WITH_VFS && ZEN_PLATFORM_WINDOWS
+
+ZEN_THIRD_PARTY_INCLUDES_START
+# include <zencore/windows.h>
+# include <projectedfslib.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+struct ProjFsProviderInterface
+{
+ virtual ~ProjFsProviderInterface() = default;
+
+ // [Mandatory] Inform the provider a directory enumeration is starting.
+ // It corresponds to PRJ_START_DIRECTORY_ENUMERATION_CB in projectedfslib.h
+ virtual HRESULT StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) = 0;
+
+ // [Mandatory] Inform the provider a directory enumeration is over.
+ // It corresponds to PRJ_END_DIRECTORY_ENUMERATION_CB in projectedfslib.h
+ virtual HRESULT EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) = 0;
+
+ // [Mandatory] Request directory enumeration information from the provider.
+ // It corresponds to PRJ_GET_DIRECTORY_ENUMERATION_CB in projectedfslib.h
+ virtual HRESULT GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) = 0;
+
+ // [Mandatory] Request meta data information for a file or directory.
+ // It corresponds to PRJ_GET_PLACEHOLDER_INFO_CB in projectedfslib.h
+ virtual HRESULT GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData) = 0;
+
+ // [Mandatory] Request the contents of a file's primary data stream.
+ // It corresponds to PRJ_GET_FILE_DATA_CB in projectedfslib.h
+ virtual HRESULT GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) = 0;
+
+ // [Optional] Deliver notifications to the provider that it has specified
+ // it wishes to receive. It corresponds to PRJ_NOTIFICATION_CB in projectedfslib.h
+ virtual HRESULT Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters);
+
+ /** This callback is optional. If the provider does not supply an implementation of this callback, ProjFS will invoke the provider's
+ directory enumeration callbacks to determine the existence of a file path in the provider's store.
+
+ The provider should use PrjFileNameCompare as the comparison routine when searching its backing store for the specified file.
+ */
+ virtual HRESULT QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+
+ virtual void CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+
+ // Setup
+
+ virtual void Run() = 0;
+
+ //////////////////////////////////////////////////////////////////////////
+ // Forwarding functions
+ //
+
+ static HRESULT StartDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId);
+ static HRESULT EndDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId);
+ static HRESULT GetDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle);
+ static HRESULT GetPlaceholderInfoCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+ static HRESULT GetFileDataCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length);
+ static HRESULT NotificationCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters);
+ static HRESULT QueryFileName_C(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+ static void CancelCommand_C(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+};
+
+} // namespace zen
+#endif
diff --git a/src/zenvfs/vfs.cpp b/src/zenvfs/vfs.cpp
new file mode 100644
index 000000000..02dbc904c
--- /dev/null
+++ b/src/zenvfs/vfs.cpp
@@ -0,0 +1,56 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenvfs/vfs.h"
+#include "vfsprovider.h"
+
+#include <zencore/iohash.h>
+#include <zencore/scopeguard.h>
+#include <zencore/string.h>
+#include <zencore/thread.h>
+
+#if ZEN_WITH_VFS
+
+namespace zen {
+
+VfsHost::VfsHost(std::string_view VfsRootPath) : m_Provider(new VfsProvider(VfsRootPath))
+{
+}
+
+VfsHost::~VfsHost()
+{
+ delete m_Provider;
+}
+
+void
+VfsHost::Initialize()
+{
+ m_Provider->Initialize();
+}
+
+void
+VfsHost::Run()
+{
+ m_Provider->Run();
+}
+
+void
+VfsHost::RequestStop()
+{
+ m_Provider->RequestStop();
+}
+
+void
+VfsHost::Cleanup()
+{
+ m_Provider->Cleanup();
+}
+
+void
+VfsHost::AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource)
+{
+ m_Provider->AddMount(Mountpoint, std::move(DataSource));
+}
+
+} // namespace zen
+
+#endif
diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp
new file mode 100644
index 000000000..a3cfe9d15
--- /dev/null
+++ b/src/zenvfs/vfsprovider.cpp
@@ -0,0 +1,794 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "vfsprovider.h"
+
+#if ZEN_WITH_VFS
+
+# include <zencore/filesystem.h>
+# include <zencore/scopeguard.h>
+
+# include <system_error>
+
+namespace zen {
+
+bool
+VfsTreeNode::IsDirectory() const
+{
+ return !GetChildren().empty() || FileSize == DirectoryMarker;
+}
+
+VfsTreeNode*
+VfsTreeNode::FindNode(std::wstring_view NodeName)
+{
+ if (auto SlashIndex = NodeName.find_first_of('\\'); SlashIndex != std::string_view::npos)
+ {
+ std::wstring_view DirName = NodeName.substr(0, SlashIndex);
+
+ if (VfsTreeNode* DirNode = FindNode(DirName))
+ {
+ return DirNode->FindNode(NodeName.substr(SlashIndex + 1));
+ }
+ }
+ else
+ {
+ ExtendableWideStringBuilder<64> NodeNameZ;
+ NodeNameZ.Append(NodeName);
+
+ PopulateVirtualNode();
+
+ for (auto& Child : GetChildren())
+ {
+ ExtendableWideStringBuilder<64> ChildName;
+ Utf8ToWide(Child->Name, ChildName);
+
+ if (PrjFileNameCompare(ChildName.c_str(), NodeNameZ.c_str()) == 0)
+ {
+ return Child.get();
+ }
+ }
+ }
+
+ return nullptr;
+}
+
+VfsTreeNode*
+VfsTreeNode::FindNode(std::string_view NodeName)
+{
+ for (auto& Child : GetChildren())
+ {
+ if (NodeName == Child->Name)
+ {
+ return Child.get();
+ }
+ }
+
+ return nullptr;
+}
+
+void
+VfsTreeNode::PopulateVirtualNode()
+{
+ if (IsDirectory() && GetChildren().empty())
+ {
+ // Populate this node via the data source
+
+ DataSource->PopulateDirectory(GetPath(), *this);
+ NormalizeTree();
+ }
+}
+
+void
+VfsTreeNode::AddFileNode(std::string_view NodeName, uint64_t InFileSize, Oid InChunkId, Ref<VfsTreeDataSource> InDataSource)
+{
+ auto SlashIndex = NodeName.find_first_of('/');
+
+ if (SlashIndex != std::string_view::npos)
+ {
+ const std::string_view DirName = NodeName.substr(0, SlashIndex);
+ const std::string_view ChildPath = NodeName.substr(SlashIndex + 1);
+
+ VfsTreeNode* DirNode = FindNode(DirName);
+
+ if (!DirNode)
+ {
+ VfsTreeNode* NewNode = AddNewChildNode();
+ NewNode->Name = std::string(DirName);
+
+ DirNode = NewNode;
+ }
+
+ DirNode->AddFileNode(ChildPath, InFileSize, InChunkId, std::move(InDataSource));
+ }
+ else
+ {
+ auto InitLeafNode = [&](VfsTreeNode& NewNode) {
+ NewNode.Name = std::string(NodeName);
+ NewNode.FileSize = InFileSize;
+ NewNode.ChunkId = InChunkId;
+ NewNode.DataSource = std::move(InDataSource);
+ };
+
+ for (auto& Child : GetChildren())
+ {
+ // Already present, just update properties
+
+ if (Child->Name == NodeName)
+ {
+ InitLeafNode(*Child);
+
+ return;
+ }
+ }
+
+ InitLeafNode(*AddNewChildNode());
+ }
+}
+
+void
+VfsTreeNode::AddVirtualNode(std::string_view NodeName, Ref<VfsTreeDataSource>&& InDataSource)
+{
+ auto SlashIndex = NodeName.find_first_of('/');
+
+ if (SlashIndex != std::string_view::npos)
+ {
+ const std::string_view DirName = NodeName.substr(0, SlashIndex);
+ const std::string_view ChildPath = NodeName.substr(SlashIndex + 1);
+
+ VfsTreeNode* DirNode = FindNode(DirName);
+
+ if (!DirNode)
+ {
+ // New directory entry (non-virtual)
+
+ DirNode = AddNewChildNode();
+ DirNode->Name = std::string(DirName);
+ DirNode->FileSize = DirectoryMarker;
+ }
+
+ DirNode->AddVirtualNode(ChildPath, std::move(InDataSource));
+ }
+ else
+ {
+ // Add "leaf" node
+
+ VfsTreeNode* LeafNode = nullptr;
+
+ for (auto& Child : GetChildren())
+ {
+ // Already present, just update properties
+
+ if (Child->Name == NodeName)
+ {
+ LeafNode = Child.get();
+ break;
+ }
+ }
+
+ if (!LeafNode)
+ {
+ LeafNode = AddNewChildNode();
+ }
+
+ LeafNode->Name = std::string(NodeName);
+ LeafNode->FileSize = DirectoryMarker;
+ LeafNode->DataSource = std::move(InDataSource);
+ }
+}
+
+static bool
+FileNameLessThan(const std::unique_ptr<VfsTreeNode>& Lhs, const std::unique_ptr<VfsTreeNode>& Rhs)
+{
+ ExtendableWideStringBuilder<64> Name1, Name2;
+
+ Utf8ToWide(Lhs->GetName(), Name1);
+ Utf8ToWide(Rhs->GetName(), Name2);
+
+ return PrjFileNameCompare(Name1.c_str(), Name2.c_str()) < 0;
+}
+
+void
+VfsTreeNode::NormalizeTree()
+{
+ // Ensure entries are in the order in which ProjFS expects them. If we
+ // don't do this then there will be spurious duplicates and things will
+ // behave strangely
+
+ std::sort(begin(ChildArray), end(ChildArray), FileNameLessThan);
+
+ for (auto& Child : GetChildren())
+ {
+ Child->NormalizeTree();
+ }
+}
+
+void
+VfsTreeNode::GetPath(StringBuilderBase& Path) const
+{
+ if (ParentNode)
+ ParentNode->GetPath(Path);
+
+ if (Path.Size())
+ {
+ Path.Append("\\");
+ }
+
+ Path.Append(Name);
+}
+
+const std::string
+VfsTreeNode::GetPath() const
+{
+ ExtendableStringBuilder<64> Path;
+ GetPath(Path);
+ return Path.ToString();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+struct VfsTree::Impl
+{
+ void SetTree(VfsTreeNode&& Tree, VfsTreeDataSource& DataProvider)
+ {
+ *m_RootNode = std::move(Tree);
+ m_RootNode->NormalizeTree();
+ m_DataProvider = &DataProvider;
+ }
+ VfsTreeNode& GetTree() { return *m_RootNode; }
+
+ void Initialize(VfsTreeDataSource& DataProvider) { m_DataProvider = &DataProvider; }
+
+ void ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length)
+ {
+ ZEN_ASSERT(m_DataProvider);
+ m_DataProvider->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length);
+ }
+
+ std::unique_ptr<VfsTreeNode> m_RootNode = std::make_unique<VfsTreeNode>(nullptr);
+ VfsTreeDataSource* m_DataProvider = nullptr;
+
+ const VfsTreeNode* FindNode(std::wstring NodeName) { return m_RootNode->FindNode(NodeName); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsTree::VfsTree() : m_Impl(new Impl)
+{
+}
+
+VfsTree::~VfsTree()
+{
+ delete m_Impl;
+}
+
+void
+VfsTree::Initialize(VfsTreeDataSource& DataProvider)
+{
+ m_Impl->Initialize(DataProvider);
+}
+
+void
+VfsTree::SetTree(VfsTreeNode&& Tree, VfsTreeDataSource& DataProvider)
+{
+ m_Impl->SetTree(std::move(Tree), DataProvider);
+}
+
+VfsTreeNode&
+VfsTree::GetTree()
+{
+ return m_Impl->GetTree();
+}
+
+VfsTreeNode*
+VfsTree::FindNode(std::wstring NodeName)
+{
+ return m_Impl->m_RootNode->FindNode(NodeName);
+}
+
+void
+VfsTree::ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length)
+{
+ m_Impl->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+static Guid
+ToGuid(const GUID* GuidPtr)
+{
+ return *reinterpret_cast<const Guid*>(GuidPtr);
+}
+
+static Guid
+PathToGuid(std::string_view Path)
+{
+ const IoHash Hash = IoHash::HashBuffer(Path.data(), Path.size());
+
+ Guid Id;
+
+ memcpy(&Id.A, &Hash.Hash[0], sizeof Id.A);
+ memcpy(&Id.B, &Hash.Hash[4], sizeof Id.B);
+ memcpy(&Id.C, &Hash.Hash[8], sizeof Id.C);
+ memcpy(&Id.D, &Hash.Hash[12], sizeof Id.D);
+
+ return Id;
+}
+
+void
+VfsProvider::EnumerationState::RestartScan()
+{
+ Iterator = Node->GetChildren().begin();
+ EndIterator = Node->GetChildren().end();
+}
+
+bool
+VfsProvider::EnumerationState::MatchesPattern(PCWSTR SearchExpression)
+{
+ return PrjFileNameMatch(GetCurrentFileName(), SearchExpression);
+}
+
+void
+VfsProvider::EnumerationState::MoveToNext()
+{
+ if (Iterator != EndIterator)
+ ++Iterator;
+}
+
+const WCHAR*
+VfsProvider::EnumerationState::GetCurrentFileName()
+{
+ CurrentName = zen::Utf8ToWide((*Iterator)->GetName());
+ return CurrentName.c_str();
+}
+
+PRJ_FILE_BASIC_INFO&
+VfsProvider::EnumerationState::GetCurrentBasicInfo()
+{
+ if ((*Iterator)->IsDirectory())
+ {
+ BasicInfo = {.IsDirectory = true, .FileAttributes = FILE_ATTRIBUTE_DIRECTORY};
+ }
+ else
+ {
+ BasicInfo = {.IsDirectory = false, .FileSize = INT64((*Iterator)->GetFileSize()), .FileAttributes = FILE_ATTRIBUTE_READONLY};
+ }
+
+ return BasicInfo;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsProvider::VfsProvider(std::string_view RootPath) : m_RootPath(RootPath)
+{
+}
+
+VfsProvider::~VfsProvider()
+{
+}
+
+void
+VfsProvider::Initialize()
+{
+ std::filesystem::path Root{m_RootPath};
+ std::filesystem::path ManifestPath = Root / ".zen_vfs";
+ bool HaveManifest = false;
+
+ if (std::filesystem::exists(Root))
+ {
+ if (!std::filesystem::is_directory(Root))
+ {
+ throw std::runtime_error("specified VFS root exists but is not a directory");
+ }
+
+ std::error_code Ec;
+ m_RootPath = WideToUtf8(CanonicalPath(Root, Ec).c_str());
+
+ if (Ec)
+ {
+ throw std::system_error(Ec);
+ }
+
+ if (std::filesystem::exists(ManifestPath))
+ {
+ FileContents ManifestData = zen::ReadFile(ManifestPath);
+ CbObject Manifest = LoadCompactBinaryObject(ManifestData.Flatten());
+
+ m_RootGuid = Manifest["id"].AsUuid();
+
+ HaveManifest = true;
+ }
+ else
+ {
+ m_RootGuid = PathToGuid(m_RootPath);
+ }
+ }
+ else
+ {
+ zen::CreateDirectories(Root);
+ m_RootGuid = PathToGuid(m_RootPath);
+ }
+
+ if (!HaveManifest)
+ {
+ CbObjectWriter Cbo;
+ Cbo << "id" << m_RootGuid;
+ Cbo << "path" << m_RootPath;
+ CbObject Manifest = Cbo.Save();
+
+ zen::WriteFile(ManifestPath, Manifest.GetBuffer().AsIoBuffer());
+ }
+
+ m_IsInitialized.test_and_set();
+
+ // Set up ProjFS root placeholder
+
+ ExtendableWideStringBuilder<256> WideRootPath;
+ zen::Utf8ToWide(m_RootPath, WideRootPath);
+
+ PRJ_PLACEHOLDER_VERSION_INFO VersionInfo{};
+
+ if (HRESULT hRes = PrjMarkDirectoryAsPlaceholder(WideRootPath.c_str(), NULL, &VersionInfo, reinterpret_cast<const GUID*>(&m_RootGuid));
+ FAILED(hRes))
+ {
+ ThrowSystemException(hRes, fmt::format("PrjMarkDirectoryAsPlaceholder call for '{}' failed", m_RootPath));
+ }
+}
+
+void
+VfsProvider::Cleanup()
+{
+ if (m_IsInitialized.test())
+ {
+ zen::CleanDirectoryExceptDotFiles(m_RootPath);
+
+ m_IsInitialized.clear();
+ }
+}
+
+void
+VfsProvider::AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource)
+{
+ ZEN_ASSERT(m_IsInitialized.test());
+
+ if (!m_Tree)
+ {
+ m_Tree = new VfsTree;
+ }
+
+ auto& Tree = m_Tree->GetTree();
+
+ if (Tree.FindNode(Mountpoint))
+ {
+ throw std::runtime_error(fmt::format("mount point '{}' already exists", Mountpoint));
+ }
+ else
+ {
+ Tree.AddVirtualNode(Mountpoint, std::move(DataSource));
+ Tree.NormalizeTree();
+ }
+}
+
+void
+VfsProvider::Run()
+{
+ const PRJ_CALLBACKS Callbacks = {.StartDirectoryEnumerationCallback = ProjFsProviderInterface::StartDirEnumCallback_C,
+ .EndDirectoryEnumerationCallback = ProjFsProviderInterface::EndDirEnumCallback_C,
+ .GetDirectoryEnumerationCallback = ProjFsProviderInterface::GetDirEnumCallback_C,
+ .GetPlaceholderInfoCallback = ProjFsProviderInterface::GetPlaceholderInfoCallback_C,
+ .GetFileDataCallback = ProjFsProviderInterface::GetFileDataCallback_C,
+ /* optional */ .QueryFileNameCallback = ProjFsProviderInterface::QueryFileName_C,
+ /* optional */ .NotificationCallback = ProjFsProviderInterface::NotificationCallback_C,
+ /* optional */ .CancelCommandCallback = nullptr /* ProjFsProviderInterface::CancelCommand_C */};
+
+ PRJ_NOTIFICATION_MAPPING Notifications[] = {
+ {.NotificationBitMask = PRJ_NOTIFY_FILE_OPENED | PRJ_NOTIFY_PRE_RENAME | PRJ_NOTIFY_PRE_DELETE, .NotificationRoot = L""}};
+
+ const PRJ_STARTVIRTUALIZING_OPTIONS VirtOptions = {.Flags = PRJ_FLAG_NONE /* PRJ_FLAG_USE_NEGATIVE_PATH_CACHE */,
+ .PoolThreadCount = 1,
+ .ConcurrentThreadCount = 1,
+ .NotificationMappings = Notifications,
+ .NotificationMappingsCount = ZEN_ARRAY_COUNT(Notifications)};
+
+ PRJ_NAMESPACE_VIRTUALIZATION_CONTEXT VirtContext{};
+
+ ExtendableWideStringBuilder<256> WideRootPath;
+ zen::Utf8ToWide(m_RootPath, WideRootPath);
+
+ if (HRESULT hRes = PrjStartVirtualizing(WideRootPath.c_str(), &Callbacks, this /* InstanceContext */, &VirtOptions, &VirtContext);
+ FAILED(hRes))
+ {
+ ThrowSystemException(hRes, "PrjStartVirtualizing failed");
+ }
+
+ auto StopVirtualizing = MakeGuard([&] {
+ PrjStopVirtualizing(VirtContext);
+ m_IsRunning.clear();
+ });
+
+ m_IsRunning.test_and_set();
+
+ while (!m_StopRunningEvent.Wait(10000 /* ms */))
+ {
+ }
+}
+
+void
+VfsProvider::RequestStop()
+{
+ m_StopRunningEvent.Set();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+HRESULT
+VfsProvider::StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ if (!m_Tree)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ const std::string Path = WideToUtf8(CallbackData->FilePathName);
+ const Guid EnumId = ToGuid(EnumerationId);
+
+ VfsTreeNode* EnumNode = &m_Tree->GetTree();
+
+ if (!Path.empty())
+ {
+ EnumNode = EnumNode->FindNode(CallbackData->FilePathName);
+ }
+
+ if (!EnumNode)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ EnumNode->PopulateVirtualNode();
+
+ auto NodeIt = EnumNode->GetChildren().begin();
+ auto EndIt = EnumNode->GetChildren().end();
+
+ RwLock::ExclusiveLockScope _(m_EnumerationsLock);
+ m_ActiveEnumerations[EnumId] = EnumerationState{.Path = Path, .Node = EnumNode, .Iterator = NodeIt, .EndIterator = EndIt};
+
+ return S_OK;
+}
+
+HRESULT
+VfsProvider::EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
+{
+ ZEN_UNUSED(CallbackData);
+
+ const Guid EnumId = ToGuid(EnumerationId);
+
+ RwLock::ExclusiveLockScope _(m_EnumerationsLock);
+ m_ActiveEnumerations.erase(EnumId);
+
+ return S_OK;
+}
+
+HRESULT
+VfsProvider::GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ const Guid EnumId = ToGuid(EnumerationId);
+
+ EnumerationState* State = nullptr;
+
+ {
+ RwLock::ExclusiveLockScope _(m_EnumerationsLock);
+
+ if (auto EnumIt = m_ActiveEnumerations.find(EnumId); EnumIt == m_ActiveEnumerations.end())
+ {
+ return E_INVALIDARG;
+ }
+ else
+ {
+ State = &EnumIt->second;
+ }
+ }
+
+ if (CallbackData->Flags & PRJ_CB_DATA_FLAG_ENUM_RESTART_SCAN)
+ {
+ State->RestartScan();
+ }
+
+ bool FilledAtLeastOne = false;
+
+ while (State->IsCurrentValid())
+ {
+ if (State->MatchesPattern(SearchExpression))
+ {
+ HRESULT hRes = PrjFillDirEntryBuffer(State->GetCurrentFileName(), &State->GetCurrentBasicInfo(), DirEntryBufferHandle);
+
+ if (hRes != S_OK)
+ {
+ if (FilledAtLeastOne)
+ {
+ return S_OK;
+ }
+ else
+ {
+ return hRes;
+ }
+ }
+
+ FilledAtLeastOne = true;
+ }
+
+ State->MoveToNext();
+ }
+
+ return S_OK;
+}
+
+HRESULT
+VfsProvider::GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ PRJ_PLACEHOLDER_INFO PlaceholderInfo;
+ memset(&PlaceholderInfo, 0, sizeof PlaceholderInfo);
+
+ std::wstring Path = CallbackData->FilePathName;
+
+ if (const VfsTreeNode* FoundNode = m_Tree->FindNode(Path))
+ {
+ if (FoundNode->IsDirectory())
+ {
+ PlaceholderInfo.FileBasicInfo = {.IsDirectory = true,
+ .FileAttributes = FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED};
+ }
+ else
+ {
+ PlaceholderInfo.FileBasicInfo = {.IsDirectory = false,
+ .FileSize = static_cast<INT64>(FoundNode->GetFileSize()),
+ .FileAttributes = FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED};
+ }
+
+ // NOTE: for correctness, we should actually provide the canonical casing for the name
+ // as stored in our backing store rather than simply providing the name which ProjFS
+ // used for the lookup!
+
+ HRESULT hRes = PrjWritePlaceholderInfo(CallbackData->NamespaceVirtualizationContext,
+ CallbackData->FilePathName,
+ &PlaceholderInfo,
+ sizeof PlaceholderInfo);
+
+ if (FAILED(hRes))
+ {
+ return hRes;
+ }
+
+ return S_OK;
+ }
+ else
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+}
+
+HRESULT
+VfsProvider::GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ const VfsTreeNode* FileNode = m_Tree->FindNode(CallbackData->FilePathName);
+
+ if (!FileNode || FileNode->IsDirectory())
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ void* DataBuffer = PrjAllocateAlignedBuffer(CallbackData->NamespaceVirtualizationContext, Length);
+ auto _ = MakeGuard([&] { PrjFreeAlignedBuffer(DataBuffer); });
+
+ if (auto Source = FileNode->GetDataSource())
+ {
+ if (const Oid ChunkId = FileNode->GetChunkId(); ChunkId != Oid::Zero)
+ {
+ Source->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length);
+ }
+ else
+ {
+ Source->ReadNamedData(FileNode->GetName(), DataBuffer, ByteOffset, Length);
+ }
+ }
+ else
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ HRESULT hRes =
+ PrjWriteFileData(CallbackData->NamespaceVirtualizationContext, &CallbackData->DataStreamId, DataBuffer, ByteOffset, Length);
+
+ if (FAILED(hRes))
+ return hRes;
+
+ return S_OK;
+}
+
+HRESULT
+VfsProvider::Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters)
+{
+ ZEN_UNUSED(CallbackData, IsDirectory, DestinationFileName, NotificationParameters);
+
+ switch (NotificationType)
+ {
+ case PRJ_NOTIFICATION_PRE_RENAME:
+ return HRESULT_FROM_WIN32(ERROR_ACCESS_DENIED);
+
+ case PRJ_NOTIFICATION_PRE_DELETE:
+ if (m_IsRunning.test())
+ {
+ // don't allow deletes while virtualizing, it doesn't seem
+ // to make sense as the operation won't be reflected into
+ // the underlying data source
+ return HRESULT_FROM_NT(0xC0000121L /* STATUS_CANNOT_DELETE */);
+ }
+ else
+ {
+ // file deletion is ok during cleanup
+ return S_OK;
+ }
+
+ default:
+ break;
+ }
+
+ return S_OK;
+}
+
+/** This callback is optional. If the provider does not supply an implementation of this callback, ProjFS will invoke the provider's
+ directory enumeration callbacks to determine the existence of a file path in the provider's store.
+
+ The provider should use PrjFileNameCompare as the comparison routine when searching its backing store for the specified file.
+ */
+
+HRESULT
+VfsProvider::QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ if (const VfsTreeNode* FileNode = m_Tree->FindNode(CallbackData->FilePathName))
+ {
+ return S_OK;
+ }
+
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+}
+
+void
+VfsProvider::CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ ZEN_UNUSED(CallbackData);
+}
+
+} // namespace zen
+
+#endif
diff --git a/src/zenvfs/vfsprovider.h b/src/zenvfs/vfsprovider.h
new file mode 100644
index 000000000..8e6896956
--- /dev/null
+++ b/src/zenvfs/vfsprovider.h
@@ -0,0 +1,112 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "projfsproviderinterface.h"
+
+#if ZEN_WITH_VFS
+
+# include <zencore/compactbinarybuilder.h>
+# include <zencore/except.h>
+# include <zencore/string.h>
+# include <zencore/thread.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+# if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+# include <projectedfslib.h>
+# endif
+ZEN_THIRD_PARTY_INCLUDES_END
+
+# include <unordered_map>
+# include <vector>
+
+namespace zen {
+
+class VfsTree
+{
+public:
+ VfsTree();
+ ~VfsTree();
+
+ void Initialize(VfsTreeDataSource& DataProvider);
+ void SetTree(VfsTreeNode&& TreeRoot, VfsTreeDataSource& DataProvider);
+ VfsTreeNode& GetTree();
+ VfsTreeNode* FindNode(std::wstring NodeName);
+ void ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length);
+
+private:
+ struct Impl;
+
+ Impl* m_Impl;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct VfsProvider : public ProjFsProviderInterface
+{
+ struct EnumerationState
+ {
+ std::string Path;
+ const VfsTreeNode* Node = nullptr;
+ std::vector<std::unique_ptr<VfsTreeNode>>::const_iterator Iterator;
+ std::vector<std::unique_ptr<VfsTreeNode>>::const_iterator EndIterator;
+
+ void RestartScan();
+ inline bool IsCurrentValid() { return Iterator != EndIterator; }
+ bool MatchesPattern(PCWSTR SearchExpression);
+ void MoveToNext();
+ const WCHAR* GetCurrentFileName();
+ PRJ_FILE_BASIC_INFO& GetCurrentBasicInfo();
+
+ PRJ_FILE_BASIC_INFO BasicInfo;
+ std::wstring CurrentName;
+ };
+
+ //////////////////////////////////////////////////////////////////////////
+
+ VfsProvider(std::string_view RootPath);
+ ~VfsProvider();
+
+ void Initialize();
+ void Cleanup();
+
+ void AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource);
+ void RequestStop();
+
+ // ProjFsProviderInterface implementation
+
+ virtual void Run() override;
+
+ virtual HRESULT StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) override;
+ virtual HRESULT EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) override;
+ virtual HRESULT GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) override;
+ virtual HRESULT GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData) override;
+ virtual HRESULT GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) override;
+ virtual HRESULT Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters) override;
+ virtual HRESULT QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData) override;
+ virtual void CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData) override;
+
+private:
+ std::string m_RootPath;
+ Guid m_RootGuid;
+ VfsTree* m_Tree = nullptr;
+
+ RwLock m_EnumerationsLock;
+ std::unordered_map<zen::Guid, EnumerationState> m_ActiveEnumerations;
+
+ Event m_StopRunningEvent;
+ std::atomic_flag m_IsRunning;
+ std::atomic_flag m_IsInitialized;
+};
+
+} // namespace zen
+
+#endif
diff --git a/src/zenvfs/xmake.lua b/src/zenvfs/xmake.lua
new file mode 100644
index 000000000..032d02219
--- /dev/null
+++ b/src/zenvfs/xmake.lua
@@ -0,0 +1,10 @@
+-- Copyright Epic Games, Inc. All Rights Reserved.
+
+target('zenvfs')
+ set_kind("static")
+ set_group("libs")
+ add_headerfiles("**.h")
+ add_files("**.cpp")
+ add_includedirs("include", {public=true})
+ add_deps("zencore")
+ add_packages("vcpkg::spdlog")
diff --git a/src/zenvfs/zenvfs.cpp b/src/zenvfs/zenvfs.cpp
new file mode 100644
index 000000000..bec7de473
--- /dev/null
+++ b/src/zenvfs/zenvfs.cpp
@@ -0,0 +1,13 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenvfs/zenvfs.h>
+
+namespace zen {
+
+// temporary dummy function to appease Mac linker
+void
+ZenVfs()
+{
+}
+
+} // namespace zen
diff --git a/xmake.lua b/xmake.lua
index ac01f4694..10118d13f 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -156,6 +156,7 @@ includes("src/zencore", "src/zencore-test")
includes("src/zenhttp")
includes("src/zenstore", "src/zenstore-test")
includes("src/zenutil")
+includes("src/zenvfs")
includes("src/zenserver", "src/zenserver-test")
includes("src/zen")
includes("src/zentest-appstub")