aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/vfs
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/vfs')
-rw-r--r--src/zenserver/vfs/vfsimpl.cpp450
-rw-r--r--src/zenserver/vfs/vfsimpl.h100
-rw-r--r--src/zenserver/vfs/vfsservice.cpp59
-rw-r--r--src/zenserver/vfs/vfsservice.h13
4 files changed, 8 insertions, 614 deletions
diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp
deleted file mode 100644
index d22738827..000000000
--- a/src/zenserver/vfs/vfsimpl.cpp
+++ /dev/null
@@ -1,450 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "vfsimpl.h"
-#include "vfsservice.h"
-
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
-#include <zenstore/cache/structuredcachestore.h>
-#include <zenvfs/projfs.h>
-#include <zenvfs/vfs.h>
-
-#include <memory>
-#include <unordered_map>
-
-#if ZEN_WITH_VFS
-
-namespace zen {
-
-using namespace std::literals;
-
-//////////////////////////////////////////////////////////////////////////
-
-VfsOplogDataSource::VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore)
-: m_ProjectId(ProjectId)
-, m_OplogId(OplogId)
-, m_ProjectStore(std::move(InProjectStore))
-{
-}
-
-void
-VfsOplogDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
-{
- ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount);
-}
-
-void
-VfsOplogDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
-{
- IoBuffer ChunkBuffer = m_ProjectStore->GetChunk(m_ProjectId, m_OplogId, ChunkId);
- if (ChunkBuffer)
- {
- ZEN_ASSERT(ChunkBuffer.GetSize() >= ByteOffset);
- ZEN_ASSERT(ChunkBuffer.GetSize() - ByteOffset >= ByteCount);
- MutableMemoryView Target(Buffer, ByteCount);
- Target.CopyFrom(ChunkBuffer.GetView().Mid(ByteOffset, ByteCount));
- }
-}
-
-void
-VfsOplogDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
-{
- // This should never be called
- ZEN_UNUSED(NodePath, DirNode);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-VfsCacheDataSource::VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore)
-: m_NamespaceId(NamespaceId)
-, m_BucketId(BucketId)
-, m_CacheStore(std::move(InCacheStore))
-{
-}
-
-VfsCacheDataSource::~VfsCacheDataSource()
-{
-}
-
-void
-VfsCacheDataSource::ReadNamedData(std::string_view Name, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
-{
- if (auto DotIndex = Name.find_first_of('.'); DotIndex != std::string_view::npos)
- {
- Name = Name.substr(0, DotIndex);
- }
-
- IoHash HashKey = IoHash::FromHexString(Name);
-
- CacheRequestContext CacheContext{};
-
- ZenCacheValue Value;
- if (m_CacheStore->Get(CacheContext, m_NamespaceId, m_BucketId, HashKey, /* out */ Value))
- {
- // TODO bounds check!
- auto DataPtr = reinterpret_cast<const uint8_t*>(Value.Value.GetData()) + ByteOffset;
-
- memcpy(Buffer, DataPtr, ByteCount);
-
- return;
- }
-}
-
-void
-VfsCacheDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
-{
- ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount);
-}
-
-void
-VfsCacheDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
-{
- ZEN_UNUSED(NodePath, DirNode);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-VfsService::Impl::Impl()
-{
-}
-
-VfsService::Impl::~Impl()
-{
- Unmount();
-}
-
-void
-VfsService::Impl::Mount(std::string_view MountPoint)
-{
- ZEN_INFO("VFS mount requested at '{}'", MountPoint);
-
-# if ZEN_PLATFORM_WINDOWS
- if (!IsProjFsAvailable())
- {
- throw std::runtime_error("Projected File System component not available");
- }
-# endif
-
- if (!m_MountpointPath.empty())
- {
- throw std::runtime_error("VFS already mounted");
- }
-
- m_MountpointPath = MountPoint;
-
- RefreshVfs();
-}
-
-void
-VfsService::Impl::Unmount()
-{
- if (m_MountpointPath.empty())
- {
- return;
- }
-
- ZEN_INFO("unmounting VFS from '{}'", m_MountpointPath);
-
- m_MountpointPath.clear();
-
- RefreshVfs();
-}
-
-void
-VfsService::Impl::AddService(Ref<ProjectStore>&& Ps)
-{
- m_ProjectStore = std::move(Ps);
-
- RefreshVfs();
-}
-
-void
-VfsService::Impl::AddService(Ref<ZenCacheStore>&& Z$)
-{
- m_ZenCacheStore = std::move(Z$);
-
- RefreshVfs();
-}
-
-void
-VfsService::Impl::RefreshVfs()
-{
- if (m_VfsHost && m_MountpointPath.empty())
- {
- m_VfsHost->RequestStop();
- m_VfsThread.join();
- m_VfsHost.reset();
- m_VfsThreadRunning.Reset();
- m_VfsDataSource = nullptr;
-
- return;
- }
-
- if (!m_VfsHost && !m_MountpointPath.empty())
- {
- m_VfsThread = std::thread(&VfsService::Impl::VfsThread, this);
- m_VfsThreadRunning.Wait();
-
- // At this stage, m_VfsHost should be initialized
-
- ZEN_ASSERT(m_VfsHost);
- }
-
- if (m_ProjectStore && m_VfsHost)
- {
- if (!m_VfsDataSource)
- {
- m_VfsDataSource = new VfsServiceDataSource(this);
- }
-
- m_VfsHost->AddMount("projects"sv, m_VfsDataSource);
- }
-
- if (m_ZenCacheStore && m_VfsHost)
- {
- if (!m_VfsDataSource)
- {
- m_VfsDataSource = new VfsServiceDataSource(this);
- }
-
- m_VfsHost->AddMount("ddc_cache"sv, m_VfsDataSource);
- }
-}
-
-void
-VfsService::Impl::VfsThread()
-{
- SetCurrentThreadName("VFS");
-
- ZEN_INFO("VFS service thread now RUNNING");
-
- try
- {
- m_VfsHost = std::make_unique<VfsHost>(m_MountpointPath);
- m_VfsHost->Initialize();
-
- m_VfsThreadRunning.Set();
- m_VfsHost->Run();
- }
- catch (const std::exception& Ex)
- {
- ZEN_WARN("exception caught in VFS thread: {}", Ex.what());
-
- m_VfsThreadException = std::current_exception();
- }
-
- if (m_VfsHost)
- {
- m_VfsHost->Cleanup();
- }
-
- ZEN_INFO("VFS service thread now EXITING");
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-Ref<VfsOplogDataSource>
-VfsServiceDataSource::GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId)
-{
- ExtendableStringBuilder<256> Key;
- Key << ProjectId << "." << OplogId;
- std::string StdKey{Key};
-
- RwLock::ExclusiveLockScope _(m_Lock);
-
- if (auto It = m_OplogSourceMap.find(StdKey); It == m_OplogSourceMap.end())
- {
- Ref<VfsOplogDataSource> NewSource{new VfsOplogDataSource(ProjectId, OplogId, m_VfsImpl->m_ProjectStore)};
- m_OplogSourceMap[StdKey] = NewSource;
- return NewSource;
- }
- else
- {
- return It->second;
- }
-}
-
-Ref<VfsCacheDataSource>
-VfsServiceDataSource::GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId)
-{
- ExtendableStringBuilder<256> Key;
- Key << NamespaceId << "." << BucketId;
- std::string StdKey{Key};
-
- RwLock::ExclusiveLockScope _(m_Lock);
-
- if (auto It = m_CacheSourceMap.find(StdKey); It == m_CacheSourceMap.end())
- {
- Ref<VfsCacheDataSource> NewSource{new VfsCacheDataSource(NamespaceId, BucketId, m_VfsImpl->m_ZenCacheStore)};
- m_CacheSourceMap[StdKey] = NewSource;
- return NewSource;
- }
- else
- {
- return It->second;
- }
-}
-
-void
-VfsServiceDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
-{
- ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount);
-}
-
-void
-VfsServiceDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
-{
- ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount);
-}
-
-void
-VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode)
-{
- if (NodePath == "projects"sv)
- {
- // Project enumeration
-
- m_VfsImpl->m_ProjectStore->DiscoverProjects();
-
- m_VfsImpl->m_ProjectStore->IterateProjects(
- [&](ProjectStore::Project& Project) { DirNode.AddVirtualNode(Project.Identifier, m_VfsImpl->m_VfsDataSource); });
- }
- else if (NodePath.starts_with("projects\\"sv))
- {
- std::string_view ProjectId{NodePath};
- ProjectId = ProjectId.substr(9); // Skip "projects\"
-
- if (std::string_view::size_type SlashOffset = ProjectId.find_first_of('\\'); SlashOffset == std::string_view::npos)
- {
- Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId);
-
- if (!Project)
- {
- // No such project found?
-
- return;
- }
-
- // Oplog enumeration
-
- std::vector<std::string> Oplogs = Project->ScanForOplogs();
-
- for (auto& Oplog : Oplogs)
- {
- DirNode.AddVirtualNode(Oplog, m_VfsImpl->m_VfsDataSource);
- }
- }
- else
- {
- std::string_view OplogId = ProjectId.substr(SlashOffset + 1);
- ProjectId = ProjectId.substr(0, SlashOffset);
-
- Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId);
-
- if (!Project)
- {
- // No such project found?
-
- return;
- }
-
- // Oplog contents enumeration
-
- if (Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true))
- {
- Ref<VfsOplogDataSource> DataSource = GetOplogDataSource(ProjectId, OplogId);
-
- // Get metadata for all chunks
- std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfos = Oplog->GetAllChunksInfo(Project->RootDir);
-
- std::unordered_map<zen::Oid, uint64_t> ChunkSizes;
-
- for (const auto& Ci : ChunkInfos)
- {
- ChunkSizes[Ci.ChunkId] = Ci.ChunkSize;
- }
-
- auto EmitFilesForDataArray = [&](zen::CbArrayView DataArray) {
- for (auto DataIter : DataArray)
- {
- if (zen::CbObjectView Data = DataIter.AsObjectView())
- {
- std::filesystem::path FileName(Data["filename"sv].AsU8String());
- zen::Oid ChunkId = Data["id"sv].AsObjectId();
-
- if (auto FindIt = ChunkSizes.find(ChunkId); FindIt != ChunkSizes.end())
- {
- DirNode.AddFileNode(FileName.string(), FindIt->second /* file size */, ChunkId, DataSource);
- }
- else
- {
- ZEN_WARN("no chunk metadata found for chunk {} (file: '{}')", ChunkId, FileName);
- }
- }
- }
- };
-
- Oplog->IterateOplog(
- [&](CbObjectView Op) {
- EmitFilesForDataArray(Op["packagedata"sv].AsArrayView());
- EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView());
- },
- ProjectStore::Oplog::Paging{});
-
- DirNode.AddFileNode("stats.json", 42, Oid::Zero);
- }
- }
- }
- else if (NodePath == "ddc_cache"sv)
- {
- // Namespace enumeration
-
- std::vector<std::string> Namespaces = m_VfsImpl->m_ZenCacheStore->GetNamespaces();
-
- for (auto& Namespace : Namespaces)
- {
- DirNode.AddVirtualNode(Namespace, m_VfsImpl->m_VfsDataSource);
- }
- }
- else if (NodePath.starts_with("ddc_cache\\"sv))
- {
- std::string_view NamespaceId{NodePath};
- NamespaceId = NamespaceId.substr(10); // Skip "ddc_cache\"
-
- auto& Cache = m_VfsImpl->m_ZenCacheStore;
-
- if (std::string_view::size_type SlashOffset = NamespaceId.find_first_of('\\'); SlashOffset == std::string_view::npos)
- {
- // Bucket enumeration
-
- if (auto NsInfo = Cache->GetNamespaceInfo(NamespaceId))
- {
- for (auto& BucketName : NsInfo->BucketNames)
- {
- DirNode.AddVirtualNode(BucketName, m_VfsImpl->m_VfsDataSource);
- }
- }
- }
- else
- {
- // Bucket contents enumeration
-
- std::string_view BucketId = NamespaceId.substr(SlashOffset + 1);
- NamespaceId = NamespaceId.substr(0, SlashOffset);
-
- Ref<VfsCacheDataSource> DataSource = GetCacheDataSource(NamespaceId, BucketId);
-
- auto Enumerator = [&](const IoHash& Key, const CacheValueDetails::ValueDetails& Details) {
- ExtendableStringBuilder<64> KeyString;
- Key.ToHexString(KeyString);
- KeyString.Append(".udd");
- DirNode.AddFileNode(KeyString, Details.Size, Oid::Zero, DataSource);
- };
-
- Cache->EnumerateBucketContents(NamespaceId, BucketId, Enumerator);
- }
- }
-}
-
-} // namespace zen
-#endif
diff --git a/src/zenserver/vfs/vfsimpl.h b/src/zenserver/vfs/vfsimpl.h
deleted file mode 100644
index 0dabf2c67..000000000
--- a/src/zenserver/vfs/vfsimpl.h
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include "vfsservice.h"
-
-#include <zencore/logging.h>
-#include <zenstore/projectstore.h>
-#include <zenvfs/vfs.h>
-
-#if ZEN_WITH_VFS
-
-# include <memory>
-# include <unordered_map>
-
-namespace zen {
-
-struct VfsOplogDataSource : public VfsTreeDataSource
-{
- VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore);
-
- virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
- virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
- virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
-
-private:
- std::string m_ProjectId;
- std::string m_OplogId;
- Ref<ProjectStore> m_ProjectStore;
-};
-
-struct VfsCacheDataSource : public VfsTreeDataSource
-{
- VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore);
- ~VfsCacheDataSource();
-
- virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
- virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
- virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
-
-private:
- std::string m_NamespaceId;
- std::string m_BucketId;
- Ref<ZenCacheStore> m_CacheStore;
-};
-
-struct VfsServiceDataSource : public VfsTreeDataSource
-{
- VfsServiceDataSource(VfsService::Impl* VfsImpl) : m_VfsImpl(VfsImpl) {}
-
- virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
- virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override;
- virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override;
-
-private:
- VfsService::Impl* m_VfsImpl = nullptr;
-
- RwLock m_Lock;
- std::unordered_map<std::string, Ref<VfsOplogDataSource>> m_OplogSourceMap;
- std::unordered_map<std::string, Ref<VfsCacheDataSource>> m_CacheSourceMap;
-
- Ref<VfsOplogDataSource> GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId);
- Ref<VfsCacheDataSource> GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId);
-};
-
-//////////////////////////////////////////////////////////////////////////
-
-struct VfsService::Impl
-{
- Impl();
- ~Impl();
-
- void Mount(std::string_view MountPoint);
- void Unmount();
- void AddService(Ref<ProjectStore>&&);
- void AddService(Ref<ZenCacheStore>&&);
-
- inline std::string GetMountpointPath() { return m_MountpointPath; }
- inline bool IsVfsRunning() const { return !!m_VfsHost.get(); }
-
-private:
- Ref<ProjectStore> m_ProjectStore;
- Ref<ZenCacheStore> m_ZenCacheStore;
- Ref<VfsServiceDataSource> m_VfsDataSource;
- std::string m_MountpointPath;
-
- std::unique_ptr<VfsHost> m_VfsHost;
- std::thread m_VfsThread;
- Event m_VfsThreadRunning;
- std::exception_ptr m_VfsThreadException;
-
- void RefreshVfs();
- void VfsThread();
-
- friend struct VfsServiceDataSource;
-};
-
-} // namespace zen
-
-#endif
diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp
index bf761f8d1..863ec348a 100644
--- a/src/zenserver/vfs/vfsservice.cpp
+++ b/src/zenserver/vfs/vfsservice.cpp
@@ -1,7 +1,8 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include "vfsservice.h"
-#include "vfsimpl.h"
+
+#include <zenstore/vfsimpl.h>
#include <zencore/compactbinarybuilder.h>
@@ -61,10 +62,8 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb)
// echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @-
// echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @-
-VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService)
+VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService), m_Impl(ServiceImpl)
{
- m_Impl = new Impl;
-
m_Router.RegisterRoute(
"info",
[&](HttpRouterRequest& Request) {
@@ -142,67 +141,19 @@ VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(Statu
VfsService::~VfsService()
{
m_StatusService.UnregisterHandler("vfs", *this);
- delete m_Impl;
-}
-
-void
-VfsService::Mount(std::string_view MountPoint)
-{
- m_Impl->Mount(MountPoint);
-}
-
-void
-VfsService::Unmount()
-{
- m_Impl->Unmount();
-}
-
-void
-VfsService::AddService(Ref<ProjectStore>&& Ps)
-{
- m_Impl->AddService(std::move(Ps));
-}
-
-void
-VfsService::AddService(Ref<ZenCacheStore>&& Z$)
-{
- m_Impl->AddService(std::move(Z$));
}
#else
-VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService)
+VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService)
{
- ZEN_UNUSED(StatusService);
+ ZEN_UNUSED(ServiceImpl);
}
VfsService::~VfsService()
{
}
-void
-VfsService::Mount(std::string_view MountPoint)
-{
- ZEN_UNUSED(MountPoint);
-}
-
-void
-VfsService::Unmount()
-{
-}
-
-void
-VfsService::AddService(Ref<ProjectStore>&& Ps)
-{
- ZEN_UNUSED(Ps);
-}
-
-void
-VfsService::AddService(Ref<ZenCacheStore>&& Z$)
-{
- ZEN_UNUSED(Z$);
-}
-
#endif
const char*
diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h
index 0d0168e23..4e06da878 100644
--- a/src/zenserver/vfs/vfsservice.h
+++ b/src/zenserver/vfs/vfsservice.h
@@ -5,7 +5,6 @@
#include <zenbase/refcount.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstatus.h>
-#include <zenvfs/vfs.h>
#include <memory>
@@ -13,6 +12,7 @@ namespace zen {
class ProjectStore;
class ZenCacheStore;
+struct VfsServiceImpl;
/** Virtual File System service
@@ -28,23 +28,16 @@ class ZenCacheStore;
class VfsService : public HttpService, public IHttpStatusProvider
{
public:
- explicit VfsService(HttpStatusService& StatusService);
+ explicit VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl);
~VfsService();
- void Mount(std::string_view MountPoint);
- void Unmount();
-
- void AddService(Ref<ProjectStore>&&);
- void AddService(Ref<ZenCacheStore>&&);
-
protected:
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override;
virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
- struct Impl;
- Impl* m_Impl = nullptr;
+ VfsServiceImpl* m_Impl = nullptr;
HttpStatusService& m_StatusService;
HttpRequestRouter m_Router;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-23 12:56:18 +0000