aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2023-09-20 15:22:03 +0200
committerGitHub <[email protected]>2023-09-20 15:22:03 +0200
commit14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f (patch)
treebf24ac15759385cea339f7e1cf5380f984f5699a /src
parentchangelog version bump (diff)
downloadzen-14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f.tar.xz
zen-14d7568f9c7d970b7bbf7b6463a0a8530f98bb6f.zip
VFS implementation for local storage service (#396)
currently, only Windows (using Projected File System) is supported
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/copy.h9
-rw-r--r--src/zen/cmds/up.h4
-rw-r--r--src/zen/cmds/vfs_cmd.cpp108
-rw-r--r--src/zen/cmds/vfs_cmd.h26
-rw-r--r--src/zen/zen.cpp3
-rw-r--r--src/zencore/filesystem.cpp75
-rw-r--r--src/zencore/include/zencore/filesystem.h8
-rw-r--r--src/zenserver/cache/cachedisklayer.cpp25
-rw-r--r--src/zenserver/cache/cachedisklayer.h3
-rw-r--r--src/zenserver/cache/structuredcachestore.cpp18
-rw-r--r--src/zenserver/cache/structuredcachestore.h14
-rw-r--r--src/zenserver/objectstore/objectstore.cpp2
-rw-r--r--src/zenserver/projectstore/projectstore.cpp110
-rw-r--r--src/zenserver/projectstore/projectstore.h17
-rw-r--r--src/zenserver/vfs/vfsimpl.cpp457
-rw-r--r--src/zenserver/vfs/vfsimpl.h100
-rw-r--r--src/zenserver/vfs/vfsservice.cpp217
-rw-r--r--src/zenserver/vfs/vfsservice.h52
-rw-r--r--src/zenserver/xmake.lua3
-rw-r--r--src/zenserver/zenserver.cpp7
-rw-r--r--src/zenvfs/include/zenvfs/projfs.h14
-rw-r--r--src/zenvfs/include/zenvfs/vfs.h89
-rw-r--r--src/zenvfs/include/zenvfs/zenvfs.h9
-rw-r--r--src/zenvfs/projfs.cpp38
-rw-r--r--src/zenvfs/projfsproviderinterface.cpp98
-rw-r--r--src/zenvfs/projfsproviderinterface.h89
-rw-r--r--src/zenvfs/vfs.cpp56
-rw-r--r--src/zenvfs/vfsprovider.cpp794
-rw-r--r--src/zenvfs/vfsprovider.h112
-rw-r--r--src/zenvfs/xmake.lua10
-rw-r--r--src/zenvfs/zenvfs.cpp13
31 files changed, 2553 insertions, 27 deletions
diff --git a/src/zen/cmds/copy.h b/src/zen/cmds/copy.h
index 5527ae9b8..549114160 100644
--- a/src/zen/cmds/copy.h
+++ b/src/zen/cmds/copy.h
@@ -18,11 +18,10 @@ public:
virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
private:
- cxxopts::Options m_Options{"copy", "Copy files"};
- std::vector<std::string> m_Positional;
- std::string m_CopySource;
- std::string m_CopyTarget;
- bool m_NoClone = false;
+ cxxopts::Options m_Options{"copy", "Copy files"};
+ std::string m_CopySource;
+ std::string m_CopyTarget;
+ bool m_NoClone = false;
};
} // namespace zen
diff --git a/src/zen/cmds/up.h b/src/zen/cmds/up.h
index 5b5c6a3f8..510cc865e 100644
--- a/src/zen/cmds/up.h
+++ b/src/zen/cmds/up.h
@@ -17,8 +17,8 @@ public:
private:
cxxopts::Options m_Options{"up", "Bring up zen service"};
- uint16_t m_Port;
- int m_OwnerPid;
+ uint16_t m_Port = 0;
+ int m_OwnerPid = 0;
std::string m_ConfigFile;
};
diff --git a/src/zen/cmds/vfs_cmd.cpp b/src/zen/cmds/vfs_cmd.cpp
new file mode 100644
index 000000000..4eb9b3195
--- /dev/null
+++ b/src/zen/cmds/vfs_cmd.cpp
@@ -0,0 +1,108 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "vfs_cmd.h"
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+#include <zenhttp/formatters.h>
+#include <zenhttp/httpclient.h>
+#include <zenutil/zenserverprocess.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+VfsCommand::VfsCommand()
+{
+ m_Options.add_option("", "", "verb", "VFS management verb (mount, unmount)", cxxopts::value(m_Verb), "<verb>");
+ m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<url>");
+ m_Options.add_option("", "", "vfs-path", "Specify VFS mount point path", cxxopts::value(m_MountPath), "<path>");
+
+ m_Options.parse_positional({"verb", "vfs-path"});
+}
+
+VfsCommand::~VfsCommand()
+{
+}
+
+int
+VfsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+ ZEN_UNUSED(GlobalOptions, argc, argv);
+
+ if (!ZenCmdBase::ParseOptions(argc, argv))
+ {
+ return 0;
+ }
+
+ // Validate arguments
+
+ m_HostName = ResolveTargetHostSpec(m_HostName);
+
+ if (m_HostName.empty())
+ throw OptionParseException("unable to resolve server specification");
+
+ HttpClient Http(m_HostName);
+
+ if (m_Verb == "mount"sv)
+ {
+ if (m_MountPath.empty())
+ throw OptionParseException("No source specified");
+
+ CbObjectWriter Cbo;
+ Cbo << "method"
+ << "mount";
+
+ Cbo.BeginObject("params");
+ Cbo << "path" << m_MountPath;
+ Cbo.EndObject();
+
+ if (HttpClient::Response Result = Http.Post("/vfs"sv, Cbo.Save()))
+ {
+ }
+ else
+ {
+ Result.ThrowError("VFS mount request failed"sv);
+
+ return 1;
+ }
+ }
+ else if (m_Verb == "unmount"sv)
+ {
+ CbObjectWriter Cbo;
+ Cbo << "method"
+ << "unmount";
+
+ if (HttpClient::Response Result = Http.Post("/vfs"sv, Cbo.Save()))
+ {
+ }
+ else
+ {
+ Result.ThrowError("VFS unmount request failed"sv);
+
+ return 1;
+ }
+ }
+ else if (m_Verb == "info"sv)
+ {
+ if (HttpClient::Response Result = Http.Get(fmt::format("/vfs/info")))
+ {
+ ExtendableStringBuilder<256> Json;
+ Result.AsObject().ToJson(Json);
+ ZEN_CONSOLE("{}", Json);
+ }
+ else
+ {
+ Result.ThrowError("VFS info fetch failed"sv);
+
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+} // namespace zen
diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h
new file mode 100644
index 000000000..35546c9b6
--- /dev/null
+++ b/src/zen/cmds/vfs_cmd.h
@@ -0,0 +1,26 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+class VfsCommand : public ZenCmdBase
+{
+public:
+ VfsCommand();
+ ~VfsCommand();
+
+ virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+ virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+ cxxopts::Options m_Options{"vfs", "Manage virtual file system"};
+
+ std::string m_Verb;
+ std::string m_HostName;
+ std::string m_MountPath;
+};
+
+} // namespace zen
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 5530b57b6..db67fbae0 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -20,6 +20,7 @@
#include "cmds/top.h"
#include "cmds/up.h"
#include "cmds/version.h"
+#include "cmds/vfs_cmd.h"
#include <zencore/filesystem.h>
#include <zencore/logging.h>
@@ -230,6 +231,7 @@ main(int argc, char** argv)
UpCommand UpCmd;
AttachCommand AttachCmd;
VersionCommand VersionCmd;
+ VfsCommand VfsCmd;
#if ZEN_WITH_TESTS
RunTestsCommand RunTestsCmd;
#endif
@@ -279,6 +281,7 @@ main(int argc, char** argv)
{"up", &UpCmd, "Bring zen server up"},
{"attach", &AttachCmd, "Add a sponsor process to a running zen service"},
{"version", &VersionCmd, "Get zen server version"},
+ {"vfs", &VfsCmd, "Manage virtual file system"},
#if ZEN_WITH_TESTS
{"runtests", &RunTestsCmd, "Run zen tests"},
#endif
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index b1ec14a37..e9f7ba72c 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -92,7 +92,7 @@ CreateDirectories(const wchar_t* Dir)
// behind
static bool
-WipeDirectory(const wchar_t* DirPath)
+WipeDirectory(const wchar_t* DirPath, bool KeepDotFiles)
{
ExtendableWideStringBuilder<128> Pattern;
Pattern.Append(DirPath);
@@ -107,7 +107,7 @@ WipeDirectory(const wchar_t* DirPath)
{
do
{
- bool IsRegular = true;
+ bool AttemptDelete = true;
if (FindData.cFileName[0] == L'.')
{
@@ -115,16 +115,21 @@ WipeDirectory(const wchar_t* DirPath)
{
if (FindData.cFileName[2] == L'\0')
{
- IsRegular = false;
+ AttemptDelete = false;
}
}
else if (FindData.cFileName[1] == L'\0')
{
- IsRegular = false;
+ AttemptDelete = false;
+ }
+
+ if (KeepDotFiles)
+ {
+ AttemptDelete = false;
}
}
- if (IsRegular)
+ if (AttemptDelete)
{
ExtendableWideStringBuilder<128> Path;
Path.Append(DirPath);
@@ -188,7 +193,8 @@ WipeDirectory(const wchar_t* DirPath)
bool
DeleteDirectories(const wchar_t* DirPath)
{
- return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE;
+ const bool KeepDotFiles = false;
+ return WipeDirectory(DirPath, KeepDotFiles) && RemoveDirectoryW(DirPath) == TRUE;
}
bool
@@ -196,7 +202,20 @@ CleanDirectory(const wchar_t* DirPath)
{
if (std::filesystem::exists(DirPath))
{
- return WipeDirectory(DirPath);
+ const bool KeepDotFiles = false;
+
+ return WipeDirectory(DirPath, KeepDotFiles);
+ }
+
+ return CreateDirectories(DirPath);
+}
+
+bool
+CleanDirectory(const wchar_t* DirPath, bool KeepDotFiles)
+{
+ if (std::filesystem::exists(DirPath))
+ {
+ return WipeDirectory(DirPath, KeepDotFiles);
}
return CreateDirectories(DirPath);
@@ -262,6 +281,19 @@ CleanDirectory(const std::filesystem::path& Dir)
#endif
}
+bool
+CleanDirectoryExceptDotFiles(const std::filesystem::path& Dir)
+{
+#if ZEN_PLATFORM_WINDOWS
+ const bool KeepDotFiles = true;
+ return CleanDirectory(Dir.c_str(), KeepDotFiles);
+#else
+ ZEN_UNUSED(Dir);
+
+ ZEN_NOT_IMPLEMENTED();
+#endif
+}
+
//////////////////////////////////////////////////////////////////////////
bool
@@ -950,7 +982,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
if (FAILED(hRes))
{
- ThrowSystemException(hRes, "Failed to open handle to volume root");
+ ThrowSystemException(hRes, fmt::format("Failed to open handle to '{}'", RootDir));
}
while (Continue)
@@ -1063,6 +1095,33 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
closedir(Dir);
#endif // ZEN_PLATFORM_WINDOWS
}
+
+std::filesystem::path
+CanonicalPath(std::filesystem::path InPath, std::error_code& Ec)
+{
+ ZEN_UNUSED(Ec);
+
+#if ZEN_PLATFORM_WINDOWS
+ windows::FileHandle Handle;
+ HRESULT hRes = Handle.Create(InPath.c_str(),
+ GENERIC_READ,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ OPEN_EXISTING,
+ FILE_FLAG_BACKUP_SEMANTICS);
+
+ if (FAILED(hRes))
+ {
+ Ec = MakeErrorCodeFromLastError();
+
+ return {};
+ }
+
+ return PathFromHandle(Handle, Ec);
+#else
+ return InPath;
+#endif
+}
+
std::filesystem::path
PathFromHandle(void* NativeHandle, std::error_code& Ec)
{
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index a78edd16b..075188993 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -30,6 +30,10 @@ ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir);
*/
ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir);
+/** Ensure directory exists and delete contents (if any) before returning
+ */
+ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir);
+
/** Map native file handle to a path
*/
ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle);
@@ -38,6 +42,10 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle);
*/
ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec);
+/** Get canonical path name from a generic path
+ */
+ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec);
+
/** Query file size from native file handle
*/
ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle);
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 7adf07350..9e6f86d79 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -1620,6 +1620,19 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilt
}
void
+ZenCacheDiskLayer::CacheBucket::EnumerateBucketContents(
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const
+{
+ RwLock::SharedLockScope _(m_IndexLock);
+ for (const auto& It : m_Index)
+ {
+ CacheValueDetails::ValueDetails Vd = GetValueDetails(It.first, It.second);
+
+ Fn(It.first, Vd);
+ }
+}
+
+void
ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx)
{
ZEN_TRACE_CPU("Z$::Disk::CollectGarbage");
@@ -2130,6 +2143,18 @@ ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
return {};
}
+void
+ZenCacheDiskLayer::EnumerateBucketContents(std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const
+{
+ RwLock::SharedLockScope _(m_Lock);
+
+ if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
+ {
+ It->second->EnumerateBucketContents(Fn);
+ }
+}
+
CacheValueDetails::NamespaceDetails
ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
{
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index 127e194f1..fc4d8cd6f 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -124,6 +124,8 @@ public:
Info GetInfo() const;
std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
+ void EnumerateBucketContents(std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
@@ -150,6 +152,7 @@ private:
uint64_t EntryCount() const;
CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
+ void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
private:
const uint64_t MaxBlockSize = 1ull << 30;
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index c8384d330..4499b05f7 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -120,6 +120,13 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket)
return AnyDropped;
}
+void
+ZenCacheNamespace::EnumerateBucketContents(std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const
+{
+ m_DiskLayer.EnumerateBucketContents(Bucket, Fn);
+}
+
bool
ZenCacheNamespace::Drop()
{
@@ -502,6 +509,17 @@ ZenCacheStore::GetValueDetails(const std::string_view NamespaceFilter,
return Details;
}
+void
+ZenCacheStore::EnumerateBucketContents(std::string_view Namespace,
+ std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>&& Fn)
+{
+ if (const ZenCacheNamespace* Ns = FindNamespace(Namespace))
+ {
+ Ns->EnumerateBucketContents(Bucket, Fn);
+ }
+}
+
ZenCacheNamespace*
ZenCacheStore::GetNamespace(std::string_view Namespace)
{
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index 8c1f995a4..239efe68f 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -69,10 +69,14 @@ public:
ZenCacheNamespace(GcManager& Gc, const std::filesystem::path& RootDir);
~ZenCacheNamespace();
- bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+
+ bool DropBucket(std::string_view Bucket);
+ void EnumerateBucketContents(std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
+
bool Drop();
- bool DropBucket(std::string_view Bucket);
void Flush();
uint64_t DiskLayerThreshold() const { return m_DiskLayerSizeThreshold; }
@@ -160,6 +164,10 @@ public:
std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket);
std::vector<std::string> GetNamespaces();
+ void EnumerateBucketContents(std::string_view Namespace,
+ std::string_view Bucket,
+ std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>&& Fn);
+
private:
const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const;
ZenCacheNamespace* GetNamespace(std::string_view Namespace);
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index e5739418e..3643e8011 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -62,7 +62,7 @@ HttpObjectStoreService::Inititalize()
[this](zen::HttpRouterRequest& Request) {
const std::string BucketName = Request.GetCapture(1);
- StringBuilder<1024> Json;
+ ExtendableStringBuilder<1024> Json;
{
CbObjectWriter Writer;
Writer.BeginArray("distributions");
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 9be600e4e..1ad4403f4 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -691,6 +691,45 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
return {};
}
+std::vector<ProjectStore::Oplog::ChunkInfo>
+ProjectStore::Oplog::GetAllChunksInfo()
+{
+ // First just capture all the chunk ids
+
+ std::vector<ChunkInfo> InfoArray;
+
+ {
+ RwLock::SharedLockScope _(m_OplogLock);
+
+ if (m_Storage)
+ {
+ const size_t NumEntries = m_FileMap.size() + m_ChunkMap.size();
+
+ InfoArray.reserve(NumEntries);
+
+ for (const auto& Kv : m_FileMap)
+ {
+ InfoArray.push_back({.ChunkId = Kv.first});
+ }
+
+ for (const auto& Kv : m_ChunkMap)
+ {
+ InfoArray.push_back({.ChunkId = Kv.first});
+ }
+ }
+ }
+
+ for (ChunkInfo& Info : InfoArray)
+ {
+ if (IoBuffer Chunk = FindChunk(Info.ChunkId))
+ {
+ Info.ChunkSize = Chunk.GetSize();
+ }
+ }
+
+ return InfoArray;
+}
+
void
ProjectStore::Oplog::IterateFileMap(
std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn)
@@ -2056,6 +2095,49 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::strin
}
std::pair<HttpResponseCode, std::string>
+ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload)
+{
+ ZEN_TRACE_CPU("ProjectStore::GetProjectChunks");
+
+ using namespace std::literals;
+
+ Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
+ if (!Project)
+ {
+ return {HttpResponseCode::NotFound, fmt::format("unknown project '{}'", ProjectId)};
+ }
+ Project->TouchProject();
+
+ ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+ if (!FoundLog)
+ {
+ return {HttpResponseCode::NotFound, fmt::format("unknown oplog '{}/{}'", ProjectId, OplogId)};
+ }
+ Project->TouchOplog(OplogId);
+
+ std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfo = FoundLog->GetAllChunksInfo();
+
+ CbObjectWriter Response;
+
+ Response.BeginArray("chunks"sv);
+ for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
+ {
+ Response << Info.ChunkId;
+ }
+ Response.EndArray();
+
+ Response.BeginArray("sizes"sv);
+ for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
+ {
+ Response << Info.ChunkSize;
+ }
+ Response.EndArray();
+
+ OutPayload = Response.Save();
+ return {HttpResponseCode::OK, {}};
+}
+
+std::pair<HttpResponseCode, std::string>
ProjectStore::GetChunkInfo(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view ChunkId,
@@ -2120,6 +2202,25 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
ZenContentType AcceptType,
IoBuffer& OutChunk)
{
+ if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
+ {
+ return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)};
+ }
+
+ const Oid Obj = Oid::FromHexString(ChunkId);
+
+ return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk);
+}
+
+std::pair<HttpResponseCode, std::string>
+ProjectStore::GetChunkRange(const std::string_view ProjectId,
+ const std::string_view OplogId,
+ Oid ChunkId,
+ uint64_t Offset,
+ uint64_t Size,
+ ZenContentType AcceptType,
+ IoBuffer& OutChunk)
+{
bool IsOffset = Offset != 0 || Size != ~(0ull);
Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
@@ -2136,14 +2237,7 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
}
Project->TouchOplog(OplogId);
- if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
- {
- return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)};
- }
-
- const Oid Obj = Oid::FromHexString(ChunkId);
-
- IoBuffer Chunk = FoundLog->FindChunk(Obj);
+ IoBuffer Chunk = FoundLog->FindChunk(ChunkId);
if (!Chunk)
{
return {HttpResponseCode::NotFound, {}};
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index aa84d04ca..9c0f8790d 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -85,6 +85,13 @@ public:
void Write();
void Update(const std::filesystem::path& MarkerPath);
+ struct ChunkInfo
+ {
+ Oid ChunkId;
+ uint64_t ChunkSize;
+ };
+
+ std::vector<ChunkInfo> GetAllChunksInfo();
void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
void IterateOplog(std::function<void(CbObject)>&& Fn);
void IterateOplogWithKey(std::function<void(int, const Oid&, CbObject)>&& Fn);
@@ -287,12 +294,22 @@ public:
const std::string_view OplogId,
bool FilterClient,
CbObject& OutPayload);
+ std::pair<HttpResponseCode, std::string> GetProjectChunks(const std::string_view ProjectId,
+ const std::string_view OplogId,
+ CbObject& OutPayload);
std::pair<HttpResponseCode, std::string> GetChunkInfo(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view ChunkId,
CbObject& OutPayload);
std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId,
const std::string_view OplogId,
+ const Oid ChunkId,
+ uint64_t Offset,
+ uint64_t Size,
+ ZenContentType AcceptType,
+ IoBuffer& OutChunk);
+ std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId,
+ const std::string_view OplogId,
const std::string_view ChunkId,
uint64_t Offset,
uint64_t Size,
diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp
new file mode 100644
index 000000000..f74000900
--- /dev/null
+++ b/src/zenserver/vfs/vfsimpl.cpp
@@ -0,0 +1,457 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "vfsimpl.h"
+#include "vfsservice.h"
+
+#include "cache/structuredcachestore.h"
+#include "projectstore/projectstore.h"
+
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zenvfs/projfs.h>
+#include <zenvfs/vfs.h>
+
+#include <memory>
+
+#if ZEN_WITH_VFS
+
+namespace zen {
+
+using namespace std::literals;
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsOplogDataSource::VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore)
+: m_ProjectId(ProjectId)
+, m_OplogId(OplogId)
+, m_ProjectStore(std::move(InProjectStore))
+{
+}
+
+void
+VfsOplogDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount);
+}
+
+void
+VfsOplogDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ IoBuffer ChunkBuffer;
+ auto Result =
+ m_ProjectStore->GetChunkRange(m_ProjectId, m_OplogId, ChunkId, 0, ~0ull, ZenContentType::kCompressedBinary, /* out */ ChunkBuffer);
+
+ if (Result.first == HttpResponseCode::OK)
+ {
+ const uint8_t* SourceBuffer = reinterpret_cast<const uint8_t*>(ChunkBuffer.GetData());
+ uint64_t AvailableBufferBytes = ChunkBuffer.GetSize();
+
+ ZEN_ASSERT(AvailableBufferBytes >= ByteOffset);
+ AvailableBufferBytes -= ByteOffset;
+ SourceBuffer += ByteOffset;
+
+ ZEN_ASSERT(AvailableBufferBytes >= ByteCount);
+ memcpy(Buffer, SourceBuffer, ByteCount);
+ }
+}
+
+void
+VfsOplogDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
+{
+ // This should never be called
+ ZEN_UNUSED(NodePath, DirNode);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsCacheDataSource::VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore)
+: m_NamespaceId(NamespaceId)
+, m_BucketId(BucketId)
+, m_CacheStore(std::move(InCacheStore))
+{
+}
+
+VfsCacheDataSource::~VfsCacheDataSource()
+{
+}
+
+void
+VfsCacheDataSource::ReadNamedData(std::string_view Name, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ if (auto DotIndex = Name.find_first_of('.'); DotIndex != std::string_view::npos)
+ {
+ Name = Name.substr(0, DotIndex);
+ }
+
+ IoHash HashKey = IoHash::FromHexString(Name);
+
+ CacheRequestContext CacheContext{};
+
+ ZenCacheValue Value;
+ if (m_CacheStore->Get(CacheContext, m_NamespaceId, m_BucketId, HashKey, /* out */ Value))
+ {
+ // TODO bounds check!
+ auto DataPtr = reinterpret_cast<const uint8_t*>(Value.Value.GetData()) + ByteOffset;
+
+ memcpy(Buffer, DataPtr, ByteCount);
+
+ return;
+ }
+}
+
+void
+VfsCacheDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount);
+}
+
+void
+VfsCacheDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
+{
+ ZEN_UNUSED(NodePath, DirNode);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsService::Impl::Impl()
+{
+}
+
+VfsService::Impl::~Impl()
+{
+ Unmount();
+}
+
+void
+VfsService::Impl::Mount(std::string_view MountPoint)
+{
+ ZEN_INFO("VFS mount requested at '{}'", MountPoint);
+
+# if ZEN_PLATFORM_WINDOWS
+ if (!IsProjFsAvailable())
+ {
+ throw std::runtime_error("Projected File System component not available");
+ }
+# endif
+
+ if (!m_MountpointPath.empty())
+ {
+ throw std::runtime_error("VFS already mounted");
+ }
+
+ m_MountpointPath = MountPoint;
+
+ RefreshVfs();
+}
+
+void
+VfsService::Impl::Unmount()
+{
+ if (m_MountpointPath.empty())
+ {
+ return;
+ }
+
+ ZEN_INFO("unmounting VFS from '{}'", m_MountpointPath);
+
+ m_MountpointPath.clear();
+
+ RefreshVfs();
+}
+
+void
+VfsService::Impl::AddService(Ref<ProjectStore>&& Ps)
+{
+ m_ProjectStore = std::move(Ps);
+
+ RefreshVfs();
+}
+
+void
+VfsService::Impl::AddService(Ref<ZenCacheStore>&& Z$)
+{
+ m_ZenCacheStore = std::move(Z$);
+
+ RefreshVfs();
+}
+
+void
+VfsService::Impl::RefreshVfs()
+{
+ if (m_VfsHost && m_MountpointPath.empty())
+ {
+ m_VfsHost->RequestStop();
+ m_VfsThread.join();
+ m_VfsHost.reset();
+ m_VfsThreadRunning.Reset();
+ m_VfsDataSource = nullptr;
+
+ return;
+ }
+
+ if (!m_VfsHost && !m_MountpointPath.empty())
+ {
+ m_VfsThread = std::thread(&VfsService::Impl::VfsThread, this);
+ m_VfsThreadRunning.Wait();
+
+ // At this stage, m_VfsHost should be initialized
+
+ ZEN_ASSERT(m_VfsHost);
+ }
+
+ if (m_ProjectStore && m_VfsHost)
+ {
+ if (!m_VfsDataSource)
+ {
+ m_VfsDataSource = new VfsServiceDataSource(this);
+ }
+
+ m_VfsHost->AddMount("projects"sv, m_VfsDataSource);
+ }
+
+ if (m_ZenCacheStore && m_VfsHost)
+ {
+ if (!m_VfsDataSource)
+ {
+ m_VfsDataSource = new VfsServiceDataSource(this);
+ }
+
+ m_VfsHost->AddMount("ddc_cache"sv, m_VfsDataSource);
+ }
+}
+
+void
+VfsService::Impl::VfsThread()
+{
+ SetCurrentThreadName("VFS");
+
+ ZEN_INFO("VFS service thread now RUNNING");
+
+ try
+ {
+ m_VfsHost = std::make_unique<VfsHost>(m_MountpointPath);
+ m_VfsHost->Initialize();
+
+ m_VfsThreadRunning.Set();
+ m_VfsHost->Run();
+ }
+ catch (std::exception& Ex)
+ {
+ ZEN_WARN("exception caught in VFS thread: {}", Ex.what());
+
+ m_VfsThreadException = std::current_exception();
+ }
+
+ if (m_VfsHost)
+ {
+ m_VfsHost->Cleanup();
+ }
+
+ ZEN_INFO("VFS service thread now EXITING");
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+Ref<VfsOplogDataSource>
+VfsServiceDataSource::GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId)
+{
+ ExtendableStringBuilder<256> Key;
+ Key << ProjectId << "." << OplogId;
+ std::string StdKey{Key};
+
+ RwLock::ExclusiveLockScope _(m_Lock);
+
+ if (auto It = m_OplogSourceMap.find(StdKey); It == m_OplogSourceMap.end())
+ {
+ Ref<VfsOplogDataSource> NewSource{new VfsOplogDataSource(ProjectId, OplogId, m_VfsImpl->m_ProjectStore)};
+ m_OplogSourceMap[StdKey] = NewSource;
+ return NewSource;
+ }
+ else
+ {
+ return It->second;
+ }
+}
+
+Ref<VfsCacheDataSource>
+VfsServiceDataSource::GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId)
+{
+ ExtendableStringBuilder<256> Key;
+ Key << NamespaceId << "." << BucketId;
+ std::string StdKey{Key};
+
+ RwLock::ExclusiveLockScope _(m_Lock);
+
+ if (auto It = m_CacheSourceMap.find(StdKey); It == m_CacheSourceMap.end())
+ {
+ Ref<VfsCacheDataSource> NewSource{new VfsCacheDataSource(NamespaceId, BucketId, m_VfsImpl->m_ZenCacheStore)};
+ m_CacheSourceMap[StdKey] = NewSource;
+ return NewSource;
+ }
+ else
+ {
+ return It->second;
+ }
+}
+
+void
+VfsServiceDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount);
+}
+
+void
+VfsServiceDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
+{
+ ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount);
+}
+
+void
+VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
+{
+ if (NodePath == "projects"sv)
+ {
+ // Project enumeration
+
+ m_VfsImpl->m_ProjectStore->DiscoverProjects();
+
+ m_VfsImpl->m_ProjectStore->IterateProjects(
+ [&](ProjectStore::Project& Project) { DirNode.AddVirtualNode(Project.Identifier, m_VfsImpl->m_VfsDataSource); });
+ }
+ else if (NodePath.starts_with("projects\\"sv))
+ {
+ std::string_view ProjectId{NodePath};
+ ProjectId = ProjectId.substr(9); // Skip "projects\"
+
+ if (std::string_view::size_type SlashOffset = ProjectId.find_first_of('\\'); SlashOffset == std::string_view::npos)
+ {
+ Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId);
+
+ if (!Project)
+ {
+ // No such project found?
+
+ return;
+ }
+
+ // Oplog enumeration
+
+ std::vector<std::string> Oplogs = Project->ScanForOplogs();
+
+ for (auto& Oplog : Oplogs)
+ {
+ DirNode.AddVirtualNode(Oplog, m_VfsImpl->m_VfsDataSource);
+ }
+ }
+ else
+ {
+ std::string_view OplogId = ProjectId.substr(SlashOffset + 1);
+ ProjectId = ProjectId.substr(0, SlashOffset);
+
+ Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId);
+
+ if (!Project)
+ {
+ // No such project found?
+
+ return;
+ }
+
+ // Oplog contents enumeration
+
+ if (ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId))
+ {
+ Ref<VfsOplogDataSource> DataSource = GetOplogDataSource(ProjectId, OplogId);
+
+ // Get metadata for all chunks
+ std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfos = Oplog->GetAllChunksInfo();
+
+ std::unordered_map<zen::Oid, uint64_t> ChunkSizes;
+
+ for (const auto& Ci : ChunkInfos)
+ {
+ ChunkSizes[Ci.ChunkId] = Ci.ChunkSize;
+ }
+
+ auto EmitFilesForDataArray = [&](zen::CbArrayView DataArray) {
+ for (auto DataIter : DataArray)
+ {
+ if (zen::CbObjectView Data = DataIter.AsObjectView())
+ {
+ std::string_view FileName = Data["filename"sv].AsString();
+ zen::Oid ChunkId = Data["id"sv].AsObjectId();
+
+ if (auto FindIt = ChunkSizes.find(ChunkId); FindIt != ChunkSizes.end())
+ {
+ DirNode.AddFileNode(FileName, FindIt->second /* file size */, ChunkId, DataSource);
+ }
+ else
+ {
+ ZEN_WARN("no chunk metadata found for chunk {} (file: '{}')", ChunkId, FileName);
+ }
+ }
+ }
+ };
+
+ Oplog->IterateOplog([&](CbObject Op) {
+ EmitFilesForDataArray(Op["packagedata"sv].AsArrayView());
+ EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView());
+ });
+
+ DirNode.AddFileNode("stats.json", 42, Oid::Zero);
+ }
+ }
+ }
+ else if (NodePath == "ddc_cache"sv)
+ {
+ // Namespace enumeration
+
+ std::vector<std::string> Namespaces = m_VfsImpl->m_ZenCacheStore->GetNamespaces();
+
+ for (auto& Namespace : Namespaces)
+ {
+ DirNode.AddVirtualNode(Namespace, m_VfsImpl->m_VfsDataSource);
+ }
+ }
+ else if (NodePath.starts_with("ddc_cache\\"sv))
+ {
+ std::string_view NamespaceId{NodePath};
+ NamespaceId = NamespaceId.substr(10); // Skip "ddc_cache\"
+
+ auto& Cache = m_VfsImpl->m_ZenCacheStore;
+
+ if (std::string_view::size_type SlashOffset = NamespaceId.find_first_of('\\'); SlashOffset == std::string_view::npos)
+ {
+ // Bucket enumeration
+
+ if (auto NsInfo = Cache->GetNamespaceInfo(NamespaceId))
+ {
+ for (auto& BucketName : NsInfo->BucketNames)
+ {
+ DirNode.AddVirtualNode(BucketName, m_VfsImpl->m_VfsDataSource);
+ }
+ }
+ }
+ else
+ {
+ // Bucket contents enumeration
+
+ std::string_view BucketId = NamespaceId.substr(SlashOffset + 1);
+ NamespaceId = NamespaceId.substr(0, SlashOffset);
+
+ Ref<VfsCacheDataSource> DataSource = GetCacheDataSource(NamespaceId, BucketId);
+
+ auto Enumerator = [&](const IoHash& Key, const CacheValueDetails::ValueDetails& Details) {
+ ExtendableStringBuilder<64> KeyString;
+ Key.ToHexString(KeyString);
+ KeyString.Append(".udd");
+ DirNode.AddFileNode(KeyString, Details.Size, Oid::Zero, DataSource);
+ };
+
+ Cache->EnumerateBucketContents(NamespaceId, BucketId, Enumerator);
+ }
+ }
+}
+
+} // namespace zen
+#endif
diff --git a/src/zenserver/vfs/vfsimpl.h b/src/zenserver/vfs/vfsimpl.h
new file mode 100644
index 000000000..9e4fdfe99
--- /dev/null
+++ b/src/zenserver/vfs/vfsimpl.h
@@ -0,0 +1,100 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "vfsservice.h"
+
+#include "projectstore/projectstore.h"
+
+#include <zencore/logging.h>
+#include <zenvfs/vfs.h>
+
+#if ZEN_WITH_VFS
+
+# include <memory>
+
+namespace zen {
+
+struct VfsOplogDataSource : public VfsTreeDataSource
+{
+ VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore);
+
+ virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
+
+private:
+ std::string m_ProjectId;
+ std::string m_OplogId;
+ Ref<ProjectStore> m_ProjectStore;
+};
+
+struct VfsCacheDataSource : public VfsTreeDataSource
+{
+ VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore);
+ ~VfsCacheDataSource();
+
+ virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
+
+private:
+ std::string m_NamespaceId;
+ std::string m_BucketId;
+ Ref<ZenCacheStore> m_CacheStore;
+};
+
+struct VfsServiceDataSource : public VfsTreeDataSource
+{
+ VfsServiceDataSource(VfsService::Impl* VfsImpl) : m_VfsImpl(VfsImpl) {}
+
+ virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
+ virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
+
+private:
+ VfsService::Impl* m_VfsImpl = nullptr;
+
+ RwLock m_Lock;
+ std::unordered_map<std::string, Ref<VfsOplogDataSource>> m_OplogSourceMap;
+ std::unordered_map<std::string, Ref<VfsCacheDataSource>> m_CacheSourceMap;
+
+ Ref<VfsOplogDataSource> GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId);
+ Ref<VfsCacheDataSource> GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId);
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct VfsService::Impl
+{
+ Impl();
+ ~Impl();
+
+ void Mount(std::string_view MountPoint);
+ void Unmount();
+ void AddService(Ref<ProjectStore>&&);
+ void AddService(Ref<ZenCacheStore>&&);
+
+ inline std::string GetMountpointPath() { return m_MountpointPath; }
+ inline bool IsVfsRunning() const { return !!m_VfsHost.get(); }
+
+private:
+ Ref<ProjectStore> m_ProjectStore;
+ Ref<ZenCacheStore> m_ZenCacheStore;
+ Ref<VfsServiceDataSource> m_VfsDataSource;
+ std::string m_MountpointPath;
+
+ std::unique_ptr<VfsHost> m_VfsHost;
+ std::thread m_VfsThread;
+ Event m_VfsThreadRunning;
+ std::exception_ptr m_VfsThreadException;
+
+ void RefreshVfs();
+ void VfsThread();
+
+ friend struct VfsServiceDataSource;
+};
+
+} // namespace zen
+
+#endif
diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp
new file mode 100644
index 000000000..c53682d93
--- /dev/null
+++ b/src/zenserver/vfs/vfsservice.cpp
@@ -0,0 +1,217 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "vfsservice.h"
+#include "vfsimpl.h"
+
+#include <zencore/compactbinarybuilder.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+#if ZEN_WITH_VFS
+
+//////////////////////////////////////////////////////////////////////////
+
+bool
+GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb)
+{
+ IoBuffer Payload = HttpReq.ReadPayload();
+ HttpContentType PayloadContentType = HttpReq.RequestContentType();
+
+ switch (PayloadContentType)
+ {
+ case HttpContentType::kJSON:
+ case HttpContentType::kUnknownContentType:
+ case HttpContentType::kText:
+ {
+ std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize());
+ Cb = LoadCompactBinaryFromJson(JsonText).AsObject();
+ if (!Cb)
+ {
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Content format not supported, expected JSON format");
+ return false;
+ }
+ }
+ break;
+ case HttpContentType::kCbObject:
+ Cb = LoadCompactBinaryObject(Payload);
+ if (!Cb)
+ {
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Content format not supported, expected compact binary format");
+ return false;
+ }
+ break;
+ default:
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+ return false;
+ }
+
+ return true;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// to test:
+//
+// echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:1337/vfs --data-binary @-
+// echo {"method": "unmount"} | curl.exe http://localhost:1337/vfs --data-binary @-
+
+VfsService::VfsService()
+{
+ m_Impl = new Impl;
+
+ m_Router.RegisterRoute(
+ "info",
+ [&](HttpRouterRequest& Request) {
+ CbObjectWriter Cbo;
+ Cbo << "running" << m_Impl->IsVfsRunning();
+ Cbo << "rootpath" << m_Impl->GetMountpointPath();
+
+ Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ },
+ HttpVerb::kGet | HttpVerb::kHead);
+
+ m_Router.RegisterRoute(
+ "",
+ [&](HttpRouterRequest& Req) {
+ CbObject Payload;
+
+ if (!GetContentAsCbObject(Req.ServerRequest(), Payload))
+ return;
+
+ std::string_view RpcName = Payload["method"sv].AsString();
+
+ if (RpcName == "mount"sv)
+ {
+ CbObjectView Params = Payload["params"sv].AsObjectView();
+ std::string_view Mountpath = Params["path"sv].AsString();
+
+ if (Mountpath.empty())
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "no path specified");
+ }
+
+ if (m_Impl->IsVfsRunning())
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS already mounted");
+ }
+
+ try
+ {
+ m_Impl->Mount(Mountpath);
+ }
+ catch (std::exception& Ex)
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what());
+ }
+
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
+ }
+ else if (RpcName == "unmount"sv)
+ {
+ if (!m_Impl->IsVfsRunning())
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS not active");
+ }
+
+ try
+ {
+ m_Impl->Unmount();
+ }
+ catch (std::exception& Ex)
+ {
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what());
+ }
+
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
+ }
+ else
+ {
+ Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "unknown RPC"sv);
+ }
+ },
+ HttpVerb::kPost);
+}
+
+VfsService::~VfsService()
+{
+ delete m_Impl;
+}
+
+void
+VfsService::Mount(std::string_view MountPoint)
+{
+ m_Impl->Mount(MountPoint);
+}
+
+void
+VfsService::Unmount()
+{
+ m_Impl->Unmount();
+}
+
+void
+VfsService::AddService(Ref<ProjectStore>&& Ps)
+{
+ m_Impl->AddService(std::move(Ps));
+}
+
+void
+VfsService::AddService(Ref<ZenCacheStore>&& Z$)
+{
+ m_Impl->AddService(std::move(Z$));
+}
+
+#else
+
+VfsService::VfsService()
+{
+}
+
+VfsService::~VfsService()
+{
+}
+
+void
+VfsService::Mount(std::string_view MountPoint)
+{
+ ZEN_UNUSED(MountPoint);
+}
+
+void
+VfsService::Unmount()
+{
+}
+
+void
+VfsService::AddService(Ref<ProjectStore>&& Ps)
+{
+ ZEN_UNUSED(Ps);
+}
+
+void
+VfsService::AddService(Ref<ZenCacheStore>&& Z$)
+{
+ ZEN_UNUSED(Z$);
+}
+
+#endif
+
+const char*
+VfsService::BaseUri() const
+{
+ return "/vfs/";
+}
+
+void
+VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest)
+{
+ m_Router.HandleRequest(HttpServiceRequest);
+}
+
+} // namespace zen
diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h
new file mode 100644
index 000000000..9510cfcda
--- /dev/null
+++ b/src/zenserver/vfs/vfsservice.h
@@ -0,0 +1,52 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/refcount.h>
+#include <zenhttp/httpserver.h>
+#include <zenvfs/vfs.h>
+
+#include <memory>
+
+namespace zen {
+
+class ProjectStore;
+class ZenCacheStore;
+
+/** Virtual File System service
+
+ Implements support for exposing data via a virtual file system interface. Currently
+ this is primarily used to surface various data stored in the local storage service
+ to users for debugging and exploration purposes.
+
+ Currently, it surfaces information from the structured cache service and from the
+ project store.
+
+ */
+
+class VfsService : public HttpService
+{
+public:
+ VfsService();
+ ~VfsService();
+
+ void Mount(std::string_view MountPoint);
+ void Unmount();
+
+ void AddService(Ref<ProjectStore>&&);
+ void AddService(Ref<ZenCacheStore>&&);
+
+protected:
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override;
+
+private:
+ struct Impl;
+ Impl* m_Impl = nullptr;
+
+ HttpRequestRouter m_Router;
+
+ friend struct VfsServiceDataSource;
+};
+
+} // namespace zen
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index a2d02baae..66bfe4ada 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -3,6 +3,7 @@
target("zenserver")
set_kind("binary")
add_deps("zencore", "zenhttp", "zenstore", "zenutil")
+ add_deps("zenvfs")
add_headerfiles("**.h")
add_files("**.cpp")
add_files("zenserver.cpp", {unity_ignored = true })
@@ -19,6 +20,8 @@ target("zenserver")
add_ldflags("/LTCG")
add_files("zenserver.rc")
add_cxxflags("/bigobj")
+ add_links("delayimp", "projectedfslib")
+ add_ldflags("/delayload:ProjectedFSLib.dll")
else
remove_files("windows/**")
end
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 988f72273..1f37e336f 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -125,6 +125,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
#include "projectstore/httpprojectstore.h"
#include "projectstore/projectstore.h"
#include "upstream/upstream.h"
+#include "vfs/vfsservice.h"
#define ZEN_APP_NAME "Zen store"
@@ -419,6 +420,11 @@ public:
m_Http->RegisterService(*m_ObjStoreService);
}
+ m_VfsService = std::make_unique<VfsService>();
+ m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore));
+ m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore));
+ m_Http->RegisterService(*m_VfsService);
+
ZEN_INFO("initializing GC, enabled '{}', interval {}s", ServerOptions.GcConfig.Enabled, ServerOptions.GcConfig.IntervalSeconds);
zen::GcSchedulerConfig GcConfig{
.RootDirectory = m_DataRoot / "gc",
@@ -750,6 +756,7 @@ private:
#endif // ZEN_WITH_COMPUTE_SERVICES
std::unique_ptr<zen::HttpFrontendService> m_FrontendService;
std::unique_ptr<zen::HttpObjectStoreService> m_ObjStoreService;
+ std::unique_ptr<zen::VfsService> m_VfsService;
std::unique_ptr<JobQueue> m_JobQueue;
std::unique_ptr<zen::HttpAdminService> m_AdminService;
diff --git a/src/zenvfs/include/zenvfs/projfs.h b/src/zenvfs/include/zenvfs/projfs.h
new file mode 100644
index 000000000..8a89c2835
--- /dev/null
+++ b/src/zenvfs/include/zenvfs/projfs.h
@@ -0,0 +1,14 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenvfs.h"
+
+#if ZEN_WITH_VFS
+
+namespace zen {
+
+bool IsProjFsAvailable();
+
+} // namespace zen
+#endif
diff --git a/src/zenvfs/include/zenvfs/vfs.h b/src/zenvfs/include/zenvfs/vfs.h
new file mode 100644
index 000000000..412970860
--- /dev/null
+++ b/src/zenvfs/include/zenvfs/vfs.h
@@ -0,0 +1,89 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenvfs.h"
+
+#if ZEN_WITH_VFS
+
+# include <zencore/zencore.h>
+# include <zencore/refcount.h>
+# include <zencore/uid.h>
+
+# include <string_view>
+# include <memory>
+# include <vector>
+
+namespace zen {
+
+struct VfsProvider;
+
+//////////////////////////////////////////////////////////////////////////
+
+struct VfsTreeNode;
+
+struct VfsTreeDataSource : public RefCounted
+{
+ virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) = 0;
+ virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) = 0;
+ virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) = 0;
+};
+
+struct VfsTreeNode
+{
+ inline VfsTreeNode(VfsTreeNode* InParent) : ParentNode(InParent) {}
+
+ const std::string_view GetName() const { return Name; }
+ uint64_t GetFileSize() const { return FileSize; }
+ VfsTreeDataSource* GetDataSource() const { return DataSource.Get(); }
+ const Oid& GetChunkId() const { return ChunkId; }
+
+ const std::string GetPath() const;
+
+ VfsTreeNode* FindNode(std::wstring_view NodeName);
+ VfsTreeNode* FindNode(std::string_view NodeName);
+
+ void AddVirtualNode(std::string_view NodeName, Ref<VfsTreeDataSource>&& DataSource);
+ void AddFileNode(std::string_view NodeName, uint64_t InFileSize, Oid InChunkId, Ref<VfsTreeDataSource> DataSource = {});
+ void NormalizeTree();
+ bool IsDirectory() const;
+ void PopulateVirtualNode();
+
+ const std::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() const { return ChildArray; }
+ std::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() { return ChildArray; }
+
+private:
+ std::string Name;
+ uint64_t FileSize = 0;
+ Oid ChunkId = Oid::Zero;
+ Ref<VfsTreeDataSource> DataSource;
+
+ inline static const uint64_t DirectoryMarker = ~uint64_t(0);
+ std::vector<std::unique_ptr<VfsTreeNode>> ChildArray;
+ VfsTreeNode* ParentNode = nullptr;
+
+ VfsTreeNode* AddNewChildNode() { return ChildArray.emplace_back(std::make_unique<VfsTreeNode>(this)).get(); }
+
+ void GetPath(StringBuilderBase& Path) const;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+class VfsHost
+{
+public:
+ VfsHost(std::string_view VfsRootPath);
+ ~VfsHost();
+
+ void Initialize();
+ void AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource);
+ void Run();
+ void RequestStop();
+ void Cleanup();
+
+private:
+ VfsProvider* m_Provider = nullptr;
+};
+
+} // namespace zen
+#endif
diff --git a/src/zenvfs/include/zenvfs/zenvfs.h b/src/zenvfs/include/zenvfs/zenvfs.h
new file mode 100644
index 000000000..c70368a1d
--- /dev/null
+++ b/src/zenvfs/include/zenvfs/zenvfs.h
@@ -0,0 +1,9 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#ifndef ZEN_WITH_VFS
+# define ZEN_WITH_VFS ZEN_PLATFORM_WINDOWS
+#endif
diff --git a/src/zenvfs/projfs.cpp b/src/zenvfs/projfs.cpp
new file mode 100644
index 000000000..5d74337b4
--- /dev/null
+++ b/src/zenvfs/projfs.cpp
@@ -0,0 +1,38 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenvfs/projfs.h"
+
+#if ZEN_WITH_VFS
+# if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+
+namespace zen {
+
+bool
+IsProjFsAvailable()
+{
+ HMODULE hMod = LoadLibraryA("projectedfslib.dll");
+
+ if (hMod)
+ {
+ FreeLibrary(hMod);
+
+ return true;
+ }
+
+ return false;
+}
+
+} // namespace zen
+# else
+namespace zen {
+
+bool
+IsProjFsAvailable()
+{
+ return false;
+}
+
+} // namespace zen
+# endif
+#endif
diff --git a/src/zenvfs/projfsproviderinterface.cpp b/src/zenvfs/projfsproviderinterface.cpp
new file mode 100644
index 000000000..ca4b91382
--- /dev/null
+++ b/src/zenvfs/projfsproviderinterface.cpp
@@ -0,0 +1,98 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "projfsproviderinterface.h"
+
+#if ZEN_WITH_VFS
+
+namespace zen {
+
+HRESULT
+ProjFsProviderInterface::Notify(const PRJ_CALLBACK_DATA* CallbackData,
+ BOOLEAN IsDirectory,
+ PRJ_NOTIFICATION NotificationType,
+ PCWSTR DestinationFileName,
+ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters)
+{
+ ZEN_UNUSED(CallbackData, IsDirectory, NotificationType, DestinationFileName, NotificationParameters);
+ return S_OK;
+}
+
+HRESULT
+ProjFsProviderInterface::QueryFileName(const PRJ_CALLBACK_DATA* CallbackData)
+{
+ ZEN_UNUSED(CallbackData);
+ return S_OK;
+}
+
+void
+ProjFsProviderInterface::CancelCommand(const PRJ_CALLBACK_DATA* CallbackData)
+{
+ ZEN_UNUSED(CallbackData);
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Forwarding functions for all ProjFS callbacks - these simply call the
+// corresponding member function on the active provider
+//
+
+HRESULT
+ProjFsProviderInterface::StartDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->StartDirEnum(CallbackData, EnumerationId);
+}
+
+HRESULT
+ProjFsProviderInterface::EndDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->EndDirEnum(CallbackData, EnumerationId);
+}
+
+HRESULT
+ProjFsProviderInterface::GetDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)
+ ->GetDirEnum(CallbackData, EnumerationId, SearchExpression, DirEntryBufferHandle);
+}
+
+HRESULT
+ProjFsProviderInterface::GetPlaceholderInfoCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->GetPlaceholderInfo(CallbackData);
+}
+
+HRESULT
+ProjFsProviderInterface::GetFileDataCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->GetFileData(CallbackData, ByteOffset, Length);
+}
+
+HRESULT
+ProjFsProviderInterface::NotificationCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)
+ ->Notify(CallbackData, IsDirectory, NotificationType, DestinationFileName, NotificationParameters);
+}
+
+HRESULT
+ProjFsProviderInterface::QueryFileName_C(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->QueryFileName(CallbackData);
+}
+
+void
+ProjFsProviderInterface::CancelCommand_C(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ return reinterpret_cast<ProjFsProviderInterface*>(CallbackData->InstanceContext)->CancelCommand(CallbackData);
+}
+
+} // namespace zen
+
+#endif
diff --git a/src/zenvfs/projfsproviderinterface.h b/src/zenvfs/projfsproviderinterface.h
new file mode 100644
index 000000000..33ea4b177
--- /dev/null
+++ b/src/zenvfs/projfsproviderinterface.h
@@ -0,0 +1,89 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenvfs/vfs.h>
+#include <zenvfs/zenvfs.h>
+
+#if ZEN_WITH_VFS && ZEN_PLATFORM_WINDOWS
+
+ZEN_THIRD_PARTY_INCLUDES_START
+# include <zencore/windows.h>
+# include <projectedfslib.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+struct ProjFsProviderInterface
+{
+ virtual ~ProjFsProviderInterface() = default;
+
+ // [Mandatory] Inform the provider a directory enumeration is starting.
+ // It corresponds to PRJ_START_DIRECTORY_ENUMERATION_CB in projectedfslib.h
+ virtual HRESULT StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) = 0;
+
+ // [Mandatory] Inform the provider a directory enumeration is over.
+ // It corresponds to PRJ_END_DIRECTORY_ENUMERATION_CB in projectedfslib.h
+ virtual HRESULT EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) = 0;
+
+ // [Mandatory] Request directory enumeration information from the provider.
+ // It corresponds to PRJ_GET_DIRECTORY_ENUMERATION_CB in projectedfslib.h
+ virtual HRESULT GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) = 0;
+
+ // [Mandatory] Request meta data information for a file or directory.
+ // It corresponds to PRJ_GET_PLACEHOLDER_INFO_CB in projectedfslib.h
+ virtual HRESULT GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData) = 0;
+
+ // [Mandatory] Request the contents of a file's primary data stream.
+ // It corresponds to PRJ_GET_FILE_DATA_CB in projectedfslib.h
+ virtual HRESULT GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) = 0;
+
+ // [Optional] Deliver notifications to the provider that it has specified
+ // it wishes to receive. It corresponds to PRJ_NOTIFICATION_CB in projectedfslib.h
+ virtual HRESULT Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters);
+
+ /** This callback is optional. If the provider does not supply an implementation of this callback, ProjFS will invoke the provider's
+ directory enumeration callbacks to determine the existence of a file path in the provider's store.
+
+ The provider should use PrjFileNameCompare as the comparison routine when searching its backing store for the specified file.
+ */
+ virtual HRESULT QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+
+ virtual void CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+
+ // Setup
+
+ virtual void Run() = 0;
+
+ //////////////////////////////////////////////////////////////////////////
+ // Forwarding functions
+ //
+
+ static HRESULT StartDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId);
+ static HRESULT EndDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId);
+ static HRESULT GetDirEnumCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle);
+ static HRESULT GetPlaceholderInfoCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+ static HRESULT GetFileDataCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length);
+ static HRESULT NotificationCallback_C(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters);
+ static HRESULT QueryFileName_C(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+ static void CancelCommand_C(_In_ const PRJ_CALLBACK_DATA* CallbackData);
+};
+
+} // namespace zen
+#endif
diff --git a/src/zenvfs/vfs.cpp b/src/zenvfs/vfs.cpp
new file mode 100644
index 000000000..02dbc904c
--- /dev/null
+++ b/src/zenvfs/vfs.cpp
@@ -0,0 +1,56 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenvfs/vfs.h"
+#include "vfsprovider.h"
+
+#include <zencore/iohash.h>
+#include <zencore/scopeguard.h>
+#include <zencore/string.h>
+#include <zencore/thread.h>
+
+#if ZEN_WITH_VFS
+
+namespace zen {
+
+VfsHost::VfsHost(std::string_view VfsRootPath) : m_Provider(new VfsProvider(VfsRootPath))
+{
+}
+
+VfsHost::~VfsHost()
+{
+ delete m_Provider;
+}
+
+void
+VfsHost::Initialize()
+{
+ m_Provider->Initialize();
+}
+
+void
+VfsHost::Run()
+{
+ m_Provider->Run();
+}
+
+void
+VfsHost::RequestStop()
+{
+ m_Provider->RequestStop();
+}
+
+void
+VfsHost::Cleanup()
+{
+ m_Provider->Cleanup();
+}
+
+void
+VfsHost::AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource)
+{
+ m_Provider->AddMount(Mountpoint, std::move(DataSource));
+}
+
+} // namespace zen
+
+#endif
diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp
new file mode 100644
index 000000000..a3cfe9d15
--- /dev/null
+++ b/src/zenvfs/vfsprovider.cpp
@@ -0,0 +1,794 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "vfsprovider.h"
+
+#if ZEN_WITH_VFS
+
+# include <zencore/filesystem.h>
+# include <zencore/scopeguard.h>
+
+# include <system_error>
+
+namespace zen {
+
+bool
+VfsTreeNode::IsDirectory() const
+{
+ return !GetChildren().empty() || FileSize == DirectoryMarker;
+}
+
+VfsTreeNode*
+VfsTreeNode::FindNode(std::wstring_view NodeName)
+{
+ if (auto SlashIndex = NodeName.find_first_of('\\'); SlashIndex != std::string_view::npos)
+ {
+ std::wstring_view DirName = NodeName.substr(0, SlashIndex);
+
+ if (VfsTreeNode* DirNode = FindNode(DirName))
+ {
+ return DirNode->FindNode(NodeName.substr(SlashIndex + 1));
+ }
+ }
+ else
+ {
+ ExtendableWideStringBuilder<64> NodeNameZ;
+ NodeNameZ.Append(NodeName);
+
+ PopulateVirtualNode();
+
+ for (auto& Child : GetChildren())
+ {
+ ExtendableWideStringBuilder<64> ChildName;
+ Utf8ToWide(Child->Name, ChildName);
+
+ if (PrjFileNameCompare(ChildName.c_str(), NodeNameZ.c_str()) == 0)
+ {
+ return Child.get();
+ }
+ }
+ }
+
+ return nullptr;
+}
+
+VfsTreeNode*
+VfsTreeNode::FindNode(std::string_view NodeName)
+{
+ for (auto& Child : GetChildren())
+ {
+ if (NodeName == Child->Name)
+ {
+ return Child.get();
+ }
+ }
+
+ return nullptr;
+}
+
+void
+VfsTreeNode::PopulateVirtualNode()
+{
+ if (IsDirectory() && GetChildren().empty())
+ {
+ // Populate this node via the data source
+
+ DataSource->PopulateDirectory(GetPath(), *this);
+ NormalizeTree();
+ }
+}
+
+void
+VfsTreeNode::AddFileNode(std::string_view NodeName, uint64_t InFileSize, Oid InChunkId, Ref<VfsTreeDataSource> InDataSource)
+{
+ auto SlashIndex = NodeName.find_first_of('/');
+
+ if (SlashIndex != std::string_view::npos)
+ {
+ const std::string_view DirName = NodeName.substr(0, SlashIndex);
+ const std::string_view ChildPath = NodeName.substr(SlashIndex + 1);
+
+ VfsTreeNode* DirNode = FindNode(DirName);
+
+ if (!DirNode)
+ {
+ VfsTreeNode* NewNode = AddNewChildNode();
+ NewNode->Name = std::string(DirName);
+
+ DirNode = NewNode;
+ }
+
+ DirNode->AddFileNode(ChildPath, InFileSize, InChunkId, std::move(InDataSource));
+ }
+ else
+ {
+ auto InitLeafNode = [&](VfsTreeNode& NewNode) {
+ NewNode.Name = std::string(NodeName);
+ NewNode.FileSize = InFileSize;
+ NewNode.ChunkId = InChunkId;
+ NewNode.DataSource = std::move(InDataSource);
+ };
+
+ for (auto& Child : GetChildren())
+ {
+ // Already present, just update properties
+
+ if (Child->Name == NodeName)
+ {
+ InitLeafNode(*Child);
+
+ return;
+ }
+ }
+
+ InitLeafNode(*AddNewChildNode());
+ }
+}
+
+void
+VfsTreeNode::AddVirtualNode(std::string_view NodeName, Ref<VfsTreeDataSource>&& InDataSource)
+{
+ auto SlashIndex = NodeName.find_first_of('/');
+
+ if (SlashIndex != std::string_view::npos)
+ {
+ const std::string_view DirName = NodeName.substr(0, SlashIndex);
+ const std::string_view ChildPath = NodeName.substr(SlashIndex + 1);
+
+ VfsTreeNode* DirNode = FindNode(DirName);
+
+ if (!DirNode)
+ {
+ // New directory entry (non-virtual)
+
+ DirNode = AddNewChildNode();
+ DirNode->Name = std::string(DirName);
+ DirNode->FileSize = DirectoryMarker;
+ }
+
+ DirNode->AddVirtualNode(ChildPath, std::move(InDataSource));
+ }
+ else
+ {
+ // Add "leaf" node
+
+ VfsTreeNode* LeafNode = nullptr;
+
+ for (auto& Child : GetChildren())
+ {
+ // Already present, just update properties
+
+ if (Child->Name == NodeName)
+ {
+ LeafNode = Child.get();
+ break;
+ }
+ }
+
+ if (!LeafNode)
+ {
+ LeafNode = AddNewChildNode();
+ }
+
+ LeafNode->Name = std::string(NodeName);
+ LeafNode->FileSize = DirectoryMarker;
+ LeafNode->DataSource = std::move(InDataSource);
+ }
+}
+
+static bool
+FileNameLessThan(const std::unique_ptr<VfsTreeNode>& Lhs, const std::unique_ptr<VfsTreeNode>& Rhs)
+{
+ ExtendableWideStringBuilder<64> Name1, Name2;
+
+ Utf8ToWide(Lhs->GetName(), Name1);
+ Utf8ToWide(Rhs->GetName(), Name2);
+
+ return PrjFileNameCompare(Name1.c_str(), Name2.c_str()) < 0;
+}
+
+void
+VfsTreeNode::NormalizeTree()
+{
+ // Ensure entries are in the order in which ProjFS expects them. If we
+ // don't do this then there will be spurious duplicates and things will
+ // behave strangely
+
+ std::sort(begin(ChildArray), end(ChildArray), FileNameLessThan);
+
+ for (auto& Child : GetChildren())
+ {
+ Child->NormalizeTree();
+ }
+}
+
+void
+VfsTreeNode::GetPath(StringBuilderBase& Path) const
+{
+ if (ParentNode)
+ ParentNode->GetPath(Path);
+
+ if (Path.Size())
+ {
+ Path.Append("\\");
+ }
+
+ Path.Append(Name);
+}
+
+const std::string
+VfsTreeNode::GetPath() const
+{
+ ExtendableStringBuilder<64> Path;
+ GetPath(Path);
+ return Path.ToString();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+struct VfsTree::Impl
+{
+ void SetTree(VfsTreeNode&& Tree, VfsTreeDataSource& DataProvider)
+ {
+ *m_RootNode = std::move(Tree);
+ m_RootNode->NormalizeTree();
+ m_DataProvider = &DataProvider;
+ }
+ VfsTreeNode& GetTree() { return *m_RootNode; }
+
+ void Initialize(VfsTreeDataSource& DataProvider) { m_DataProvider = &DataProvider; }
+
+ void ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length)
+ {
+ ZEN_ASSERT(m_DataProvider);
+ m_DataProvider->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length);
+ }
+
+ std::unique_ptr<VfsTreeNode> m_RootNode = std::make_unique<VfsTreeNode>(nullptr);
+ VfsTreeDataSource* m_DataProvider = nullptr;
+
+ const VfsTreeNode* FindNode(std::wstring NodeName) { return m_RootNode->FindNode(NodeName); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsTree::VfsTree() : m_Impl(new Impl)
+{
+}
+
+VfsTree::~VfsTree()
+{
+ delete m_Impl;
+}
+
+void
+VfsTree::Initialize(VfsTreeDataSource& DataProvider)
+{
+ m_Impl->Initialize(DataProvider);
+}
+
+void
+VfsTree::SetTree(VfsTreeNode&& Tree, VfsTreeDataSource& DataProvider)
+{
+ m_Impl->SetTree(std::move(Tree), DataProvider);
+}
+
+VfsTreeNode&
+VfsTree::GetTree()
+{
+ return m_Impl->GetTree();
+}
+
+VfsTreeNode*
+VfsTree::FindNode(std::wstring NodeName)
+{
+ return m_Impl->m_RootNode->FindNode(NodeName);
+}
+
+void
+VfsTree::ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length)
+{
+ m_Impl->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+static Guid
+ToGuid(const GUID* GuidPtr)
+{
+ return *reinterpret_cast<const Guid*>(GuidPtr);
+}
+
+static Guid
+PathToGuid(std::string_view Path)
+{
+ const IoHash Hash = IoHash::HashBuffer(Path.data(), Path.size());
+
+ Guid Id;
+
+ memcpy(&Id.A, &Hash.Hash[0], sizeof Id.A);
+ memcpy(&Id.B, &Hash.Hash[4], sizeof Id.B);
+ memcpy(&Id.C, &Hash.Hash[8], sizeof Id.C);
+ memcpy(&Id.D, &Hash.Hash[12], sizeof Id.D);
+
+ return Id;
+}
+
+void
+VfsProvider::EnumerationState::RestartScan()
+{
+ Iterator = Node->GetChildren().begin();
+ EndIterator = Node->GetChildren().end();
+}
+
+bool
+VfsProvider::EnumerationState::MatchesPattern(PCWSTR SearchExpression)
+{
+ return PrjFileNameMatch(GetCurrentFileName(), SearchExpression);
+}
+
+void
+VfsProvider::EnumerationState::MoveToNext()
+{
+ if (Iterator != EndIterator)
+ ++Iterator;
+}
+
+const WCHAR*
+VfsProvider::EnumerationState::GetCurrentFileName()
+{
+ CurrentName = zen::Utf8ToWide((*Iterator)->GetName());
+ return CurrentName.c_str();
+}
+
+PRJ_FILE_BASIC_INFO&
+VfsProvider::EnumerationState::GetCurrentBasicInfo()
+{
+ if ((*Iterator)->IsDirectory())
+ {
+ BasicInfo = {.IsDirectory = true, .FileAttributes = FILE_ATTRIBUTE_DIRECTORY};
+ }
+ else
+ {
+ BasicInfo = {.IsDirectory = false, .FileSize = INT64((*Iterator)->GetFileSize()), .FileAttributes = FILE_ATTRIBUTE_READONLY};
+ }
+
+ return BasicInfo;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+VfsProvider::VfsProvider(std::string_view RootPath) : m_RootPath(RootPath)
+{
+}
+
+VfsProvider::~VfsProvider()
+{
+}
+
+void
+VfsProvider::Initialize()
+{
+ std::filesystem::path Root{m_RootPath};
+ std::filesystem::path ManifestPath = Root / ".zen_vfs";
+ bool HaveManifest = false;
+
+ if (std::filesystem::exists(Root))
+ {
+ if (!std::filesystem::is_directory(Root))
+ {
+ throw std::runtime_error("specified VFS root exists but is not a directory");
+ }
+
+ std::error_code Ec;
+ m_RootPath = WideToUtf8(CanonicalPath(Root, Ec).c_str());
+
+ if (Ec)
+ {
+ throw std::system_error(Ec);
+ }
+
+ if (std::filesystem::exists(ManifestPath))
+ {
+ FileContents ManifestData = zen::ReadFile(ManifestPath);
+ CbObject Manifest = LoadCompactBinaryObject(ManifestData.Flatten());
+
+ m_RootGuid = Manifest["id"].AsUuid();
+
+ HaveManifest = true;
+ }
+ else
+ {
+ m_RootGuid = PathToGuid(m_RootPath);
+ }
+ }
+ else
+ {
+ zen::CreateDirectories(Root);
+ m_RootGuid = PathToGuid(m_RootPath);
+ }
+
+ if (!HaveManifest)
+ {
+ CbObjectWriter Cbo;
+ Cbo << "id" << m_RootGuid;
+ Cbo << "path" << m_RootPath;
+ CbObject Manifest = Cbo.Save();
+
+ zen::WriteFile(ManifestPath, Manifest.GetBuffer().AsIoBuffer());
+ }
+
+ m_IsInitialized.test_and_set();
+
+ // Set up ProjFS root placeholder
+
+ ExtendableWideStringBuilder<256> WideRootPath;
+ zen::Utf8ToWide(m_RootPath, WideRootPath);
+
+ PRJ_PLACEHOLDER_VERSION_INFO VersionInfo{};
+
+ if (HRESULT hRes = PrjMarkDirectoryAsPlaceholder(WideRootPath.c_str(), NULL, &VersionInfo, reinterpret_cast<const GUID*>(&m_RootGuid));
+ FAILED(hRes))
+ {
+ ThrowSystemException(hRes, fmt::format("PrjMarkDirectoryAsPlaceholder call for '{}' failed", m_RootPath));
+ }
+}
+
+void
+VfsProvider::Cleanup()
+{
+ if (m_IsInitialized.test())
+ {
+ zen::CleanDirectoryExceptDotFiles(m_RootPath);
+
+ m_IsInitialized.clear();
+ }
+}
+
+void
+VfsProvider::AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource)
+{
+ ZEN_ASSERT(m_IsInitialized.test());
+
+ if (!m_Tree)
+ {
+ m_Tree = new VfsTree;
+ }
+
+ auto& Tree = m_Tree->GetTree();
+
+ if (Tree.FindNode(Mountpoint))
+ {
+ throw std::runtime_error(fmt::format("mount point '{}' already exists", Mountpoint));
+ }
+ else
+ {
+ Tree.AddVirtualNode(Mountpoint, std::move(DataSource));
+ Tree.NormalizeTree();
+ }
+}
+
+void
+VfsProvider::Run()
+{
+ const PRJ_CALLBACKS Callbacks = {.StartDirectoryEnumerationCallback = ProjFsProviderInterface::StartDirEnumCallback_C,
+ .EndDirectoryEnumerationCallback = ProjFsProviderInterface::EndDirEnumCallback_C,
+ .GetDirectoryEnumerationCallback = ProjFsProviderInterface::GetDirEnumCallback_C,
+ .GetPlaceholderInfoCallback = ProjFsProviderInterface::GetPlaceholderInfoCallback_C,
+ .GetFileDataCallback = ProjFsProviderInterface::GetFileDataCallback_C,
+ /* optional */ .QueryFileNameCallback = ProjFsProviderInterface::QueryFileName_C,
+ /* optional */ .NotificationCallback = ProjFsProviderInterface::NotificationCallback_C,
+ /* optional */ .CancelCommandCallback = nullptr /* ProjFsProviderInterface::CancelCommand_C */};
+
+ PRJ_NOTIFICATION_MAPPING Notifications[] = {
+ {.NotificationBitMask = PRJ_NOTIFY_FILE_OPENED | PRJ_NOTIFY_PRE_RENAME | PRJ_NOTIFY_PRE_DELETE, .NotificationRoot = L""}};
+
+ const PRJ_STARTVIRTUALIZING_OPTIONS VirtOptions = {.Flags = PRJ_FLAG_NONE /* PRJ_FLAG_USE_NEGATIVE_PATH_CACHE */,
+ .PoolThreadCount = 1,
+ .ConcurrentThreadCount = 1,
+ .NotificationMappings = Notifications,
+ .NotificationMappingsCount = ZEN_ARRAY_COUNT(Notifications)};
+
+ PRJ_NAMESPACE_VIRTUALIZATION_CONTEXT VirtContext{};
+
+ ExtendableWideStringBuilder<256> WideRootPath;
+ zen::Utf8ToWide(m_RootPath, WideRootPath);
+
+ if (HRESULT hRes = PrjStartVirtualizing(WideRootPath.c_str(), &Callbacks, this /* InstanceContext */, &VirtOptions, &VirtContext);
+ FAILED(hRes))
+ {
+ ThrowSystemException(hRes, "PrjStartVirtualizing failed");
+ }
+
+ auto StopVirtualizing = MakeGuard([&] {
+ PrjStopVirtualizing(VirtContext);
+ m_IsRunning.clear();
+ });
+
+ m_IsRunning.test_and_set();
+
+ while (!m_StopRunningEvent.Wait(10000 /* ms */))
+ {
+ }
+}
+
+void
+VfsProvider::RequestStop()
+{
+ m_StopRunningEvent.Set();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+HRESULT
+VfsProvider::StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ if (!m_Tree)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ const std::string Path = WideToUtf8(CallbackData->FilePathName);
+ const Guid EnumId = ToGuid(EnumerationId);
+
+ VfsTreeNode* EnumNode = &m_Tree->GetTree();
+
+ if (!Path.empty())
+ {
+ EnumNode = EnumNode->FindNode(CallbackData->FilePathName);
+ }
+
+ if (!EnumNode)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ EnumNode->PopulateVirtualNode();
+
+ auto NodeIt = EnumNode->GetChildren().begin();
+ auto EndIt = EnumNode->GetChildren().end();
+
+ RwLock::ExclusiveLockScope _(m_EnumerationsLock);
+ m_ActiveEnumerations[EnumId] = EnumerationState{.Path = Path, .Node = EnumNode, .Iterator = NodeIt, .EndIterator = EndIt};
+
+ return S_OK;
+}
+
+HRESULT
+VfsProvider::EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId)
+{
+ ZEN_UNUSED(CallbackData);
+
+ const Guid EnumId = ToGuid(EnumerationId);
+
+ RwLock::ExclusiveLockScope _(m_EnumerationsLock);
+ m_ActiveEnumerations.erase(EnumId);
+
+ return S_OK;
+}
+
+HRESULT
+VfsProvider::GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ const Guid EnumId = ToGuid(EnumerationId);
+
+ EnumerationState* State = nullptr;
+
+ {
+ RwLock::ExclusiveLockScope _(m_EnumerationsLock);
+
+ if (auto EnumIt = m_ActiveEnumerations.find(EnumId); EnumIt == m_ActiveEnumerations.end())
+ {
+ return E_INVALIDARG;
+ }
+ else
+ {
+ State = &EnumIt->second;
+ }
+ }
+
+ if (CallbackData->Flags & PRJ_CB_DATA_FLAG_ENUM_RESTART_SCAN)
+ {
+ State->RestartScan();
+ }
+
+ bool FilledAtLeastOne = false;
+
+ while (State->IsCurrentValid())
+ {
+ if (State->MatchesPattern(SearchExpression))
+ {
+ HRESULT hRes = PrjFillDirEntryBuffer(State->GetCurrentFileName(), &State->GetCurrentBasicInfo(), DirEntryBufferHandle);
+
+ if (hRes != S_OK)
+ {
+ if (FilledAtLeastOne)
+ {
+ return S_OK;
+ }
+ else
+ {
+ return hRes;
+ }
+ }
+
+ FilledAtLeastOne = true;
+ }
+
+ State->MoveToNext();
+ }
+
+ return S_OK;
+}
+
+HRESULT
+VfsProvider::GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ PRJ_PLACEHOLDER_INFO PlaceholderInfo;
+ memset(&PlaceholderInfo, 0, sizeof PlaceholderInfo);
+
+ std::wstring Path = CallbackData->FilePathName;
+
+ if (const VfsTreeNode* FoundNode = m_Tree->FindNode(Path))
+ {
+ if (FoundNode->IsDirectory())
+ {
+ PlaceholderInfo.FileBasicInfo = {.IsDirectory = true,
+ .FileAttributes = FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED};
+ }
+ else
+ {
+ PlaceholderInfo.FileBasicInfo = {.IsDirectory = false,
+ .FileSize = static_cast<INT64>(FoundNode->GetFileSize()),
+ .FileAttributes = FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED};
+ }
+
+ // NOTE: for correctness, we should actually provide the canonical casing for the name
+ // as stored in our backing store rather than simply providing the name which ProjFS
+ // used for the lookup!
+
+ HRESULT hRes = PrjWritePlaceholderInfo(CallbackData->NamespaceVirtualizationContext,
+ CallbackData->FilePathName,
+ &PlaceholderInfo,
+ sizeof PlaceholderInfo);
+
+ if (FAILED(hRes))
+ {
+ return hRes;
+ }
+
+ return S_OK;
+ }
+ else
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+}
+
+HRESULT
+VfsProvider::GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ const VfsTreeNode* FileNode = m_Tree->FindNode(CallbackData->FilePathName);
+
+ if (!FileNode || FileNode->IsDirectory())
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ void* DataBuffer = PrjAllocateAlignedBuffer(CallbackData->NamespaceVirtualizationContext, Length);
+ auto _ = MakeGuard([&] { PrjFreeAlignedBuffer(DataBuffer); });
+
+ if (auto Source = FileNode->GetDataSource())
+ {
+ if (const Oid ChunkId = FileNode->GetChunkId(); ChunkId != Oid::Zero)
+ {
+ Source->ReadChunkData(ChunkId, DataBuffer, ByteOffset, Length);
+ }
+ else
+ {
+ Source->ReadNamedData(FileNode->GetName(), DataBuffer, ByteOffset, Length);
+ }
+ }
+ else
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ HRESULT hRes =
+ PrjWriteFileData(CallbackData->NamespaceVirtualizationContext, &CallbackData->DataStreamId, DataBuffer, ByteOffset, Length);
+
+ if (FAILED(hRes))
+ return hRes;
+
+ return S_OK;
+}
+
+HRESULT
+VfsProvider::Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters)
+{
+ ZEN_UNUSED(CallbackData, IsDirectory, DestinationFileName, NotificationParameters);
+
+ switch (NotificationType)
+ {
+ case PRJ_NOTIFICATION_PRE_RENAME:
+ return HRESULT_FROM_WIN32(ERROR_ACCESS_DENIED);
+
+ case PRJ_NOTIFICATION_PRE_DELETE:
+ if (m_IsRunning.test())
+ {
+ // don't allow deletes while virtualizing, it doesn't seem
+ // to make sense as the operation won't be reflected into
+ // the underlying data source
+ return HRESULT_FROM_NT(0xC0000121L /* STATUS_CANNOT_DELETE */);
+ }
+ else
+ {
+ // file deletion is ok during cleanup
+ return S_OK;
+ }
+
+ default:
+ break;
+ }
+
+ return S_OK;
+}
+
+/** This callback is optional. If the provider does not supply an implementation of this callback, ProjFS will invoke the provider's
+ directory enumeration callbacks to determine the existence of a file path in the provider's store.
+
+ The provider should use PrjFileNameCompare as the comparison routine when searching its backing store for the specified file.
+ */
+
+HRESULT
+VfsProvider::QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ if (m_IsRunning.test() == false)
+ {
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ }
+
+ if (const VfsTreeNode* FileNode = m_Tree->FindNode(CallbackData->FilePathName))
+ {
+ return S_OK;
+ }
+
+ return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+}
+
+void
+VfsProvider::CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData)
+{
+ ZEN_UNUSED(CallbackData);
+}
+
+} // namespace zen
+
+#endif
diff --git a/src/zenvfs/vfsprovider.h b/src/zenvfs/vfsprovider.h
new file mode 100644
index 000000000..8e6896956
--- /dev/null
+++ b/src/zenvfs/vfsprovider.h
@@ -0,0 +1,112 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "projfsproviderinterface.h"
+
+#if ZEN_WITH_VFS
+
+# include <zencore/compactbinarybuilder.h>
+# include <zencore/except.h>
+# include <zencore/string.h>
+# include <zencore/thread.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+# if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+# include <projectedfslib.h>
+# endif
+ZEN_THIRD_PARTY_INCLUDES_END
+
+# include <unordered_map>
+# include <vector>
+
+namespace zen {
+
+class VfsTree
+{
+public:
+ VfsTree();
+ ~VfsTree();
+
+ void Initialize(VfsTreeDataSource& DataProvider);
+ void SetTree(VfsTreeNode&& TreeRoot, VfsTreeDataSource& DataProvider);
+ VfsTreeNode& GetTree();
+ VfsTreeNode* FindNode(std::wstring NodeName);
+ void ReadChunkData(Oid ChunkId, void* DataBuffer, uint64_t ByteOffset, uint64_t Length);
+
+private:
+ struct Impl;
+
+ Impl* m_Impl;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct VfsProvider : public ProjFsProviderInterface
+{
+ struct EnumerationState
+ {
+ std::string Path;
+ const VfsTreeNode* Node = nullptr;
+ std::vector<std::unique_ptr<VfsTreeNode>>::const_iterator Iterator;
+ std::vector<std::unique_ptr<VfsTreeNode>>::const_iterator EndIterator;
+
+ void RestartScan();
+ inline bool IsCurrentValid() { return Iterator != EndIterator; }
+ bool MatchesPattern(PCWSTR SearchExpression);
+ void MoveToNext();
+ const WCHAR* GetCurrentFileName();
+ PRJ_FILE_BASIC_INFO& GetCurrentBasicInfo();
+
+ PRJ_FILE_BASIC_INFO BasicInfo;
+ std::wstring CurrentName;
+ };
+
+ //////////////////////////////////////////////////////////////////////////
+
+ VfsProvider(std::string_view RootPath);
+ ~VfsProvider();
+
+ void Initialize();
+ void Cleanup();
+
+ void AddMount(std::string_view Mountpoint, Ref<VfsTreeDataSource>&& DataSource);
+ void RequestStop();
+
+ // ProjFsProviderInterface implementation
+
+ virtual void Run() override;
+
+ virtual HRESULT StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) override;
+ virtual HRESULT EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) override;
+ virtual HRESULT GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ const GUID* EnumerationId,
+ _In_opt_ PCWSTR SearchExpression,
+ _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) override;
+ virtual HRESULT GetPlaceholderInfo(_In_ const PRJ_CALLBACK_DATA* CallbackData) override;
+ virtual HRESULT GetFileData(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) override;
+ virtual HRESULT Notify(_In_ const PRJ_CALLBACK_DATA* CallbackData,
+ _In_ BOOLEAN IsDirectory,
+ _In_ PRJ_NOTIFICATION NotificationType,
+ _In_opt_ PCWSTR DestinationFileName,
+ _Inout_ PRJ_NOTIFICATION_PARAMETERS* NotificationParameters) override;
+ virtual HRESULT QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData) override;
+ virtual void CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData) override;
+
+private:
+ std::string m_RootPath;
+ Guid m_RootGuid;
+ VfsTree* m_Tree = nullptr;
+
+ RwLock m_EnumerationsLock;
+ std::unordered_map<zen::Guid, EnumerationState> m_ActiveEnumerations;
+
+ Event m_StopRunningEvent;
+ std::atomic_flag m_IsRunning;
+ std::atomic_flag m_IsInitialized;
+};
+
+} // namespace zen
+
+#endif
diff --git a/src/zenvfs/xmake.lua b/src/zenvfs/xmake.lua
new file mode 100644
index 000000000..032d02219
--- /dev/null
+++ b/src/zenvfs/xmake.lua
@@ -0,0 +1,10 @@
+-- Copyright Epic Games, Inc. All Rights Reserved.
+
+target('zenvfs')
+ set_kind("static")
+ set_group("libs")
+ add_headerfiles("**.h")
+ add_files("**.cpp")
+ add_includedirs("include", {public=true})
+ add_deps("zencore")
+ add_packages("vcpkg::spdlog")
diff --git a/src/zenvfs/zenvfs.cpp b/src/zenvfs/zenvfs.cpp
new file mode 100644
index 000000000..bec7de473
--- /dev/null
+++ b/src/zenvfs/zenvfs.cpp
@@ -0,0 +1,13 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenvfs/zenvfs.h>
+
+namespace zen {
+
+// temporary dummy function to appease Mac linker
+void
+ZenVfs()
+{
+}
+
+} // namespace zen