aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenstore')
-rw-r--r--src/zenstore/blockstore.cpp14
-rw-r--r--src/zenstore/cache/cachedisklayer.cpp4
-rw-r--r--src/zenstore/cache/cacherpc.cpp69
-rw-r--r--src/zenstore/cache/structuredcachestore.cpp2
-rw-r--r--src/zenstore/chunkedfile.cpp6
-rw-r--r--src/zenstore/compactcas.cpp38
-rw-r--r--src/zenstore/filecas.cpp15
-rw-r--r--src/zenstore/gc.cpp22
-rw-r--r--src/zenstore/include/zenstore/blockstore.h2
-rw-r--r--src/zenstore/include/zenstore/caslog.h2
-rw-r--r--src/zenstore/workspaces.cpp98
11 files changed, 167 insertions, 105 deletions
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 9ad672060..e976c061d 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -2,6 +2,7 @@
#include <zenstore/blockstore.h>
+#include <zencore/enumflags.h>
#include <zencore/except.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
@@ -809,8 +810,8 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation> ChunkLocations,
ZEN_ASSERT(BlockFile);
InsertLock.ReleaseNow();
- IoBuffer ReadBuffer{IterateSmallChunkWindowSize};
- void* BufferBase = ReadBuffer.MutableData();
+ IoBuffer ReadBuffer;
+ void* BufferBase = nullptr;
size_t LocationIndexOffset = 0;
while (LocationIndexOffset < ChunkIndexes.size())
@@ -825,6 +826,11 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation> ChunkLocations,
size_t LastChunkIndex = ChunkIndexes[LocationIndexOffset + RangeCount - 1];
const BlockStoreLocation& LastLocation = ChunkLocations[LastChunkIndex];
uint64_t Size = LastLocation.Offset + LastLocation.Size - FirstLocation.Offset;
+ if (ReadBuffer.GetSize() < Size)
+ {
+ ReadBuffer = IoBuffer(Min(Size * 2, IterateSmallChunkWindowSize));
+ BufferBase = ReadBuffer.MutableData();
+ }
BlockFile->Read(BufferBase, Size, FirstLocation.Offset);
for (size_t RangeIndex = 0; RangeIndex < RangeCount; ++RangeIndex)
{
@@ -1422,8 +1428,8 @@ namespace blockstore::impl {
{
DirectoryContent DirectoryContent;
GetDirectoryContent(RootDir,
- DirectoryContent::RecursiveFlag | (Files ? DirectoryContent::IncludeFilesFlag : 0) |
- (Directories ? DirectoryContent::IncludeDirsFlag : 0),
+ DirectoryContentFlags::Recursive | (Files ? DirectoryContentFlags::IncludeFiles : DirectoryContentFlags::None) |
+ (Directories ? DirectoryContentFlags::IncludeDirs : DirectoryContentFlags::None),
DirectoryContent);
std::vector<std::filesystem::path> Result;
Result.insert(Result.end(), DirectoryContent.Directories.begin(), DirectoryContent.Directories.end());
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 9f09713ee..25f68330a 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -1267,6 +1267,7 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept
size_t IndexOffset = 0;
m_BlockStore.WriteChunks(Batch->Buffers, m_Configuration.PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) {
+ ZEN_MEMSCOPE(GetCacheDiskTag());
std::vector<DiskIndexEntry> DiskEntries;
{
RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
@@ -2679,6 +2680,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey,
Value.Value.Size(),
m_Configuration.PayloadAlignment,
[&](const BlockStoreLocation& BlockStoreLocation) {
+ ZEN_MEMSCOPE(GetCacheDiskTag());
ZEN_TRACE_CPU("Z$::Bucket::UpdateLocation");
DiskLocation Location(BlockStoreLocation, m_Configuration.PayloadAlignment, EntryFlags);
m_SlogFile.Append({.Key = HashKey, .Location = Location});
@@ -3856,7 +3858,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
ZEN_TRACE_CPU("Z$::DiscoverBuckets");
DirectoryContent DirContent;
- GetDirectoryContent(m_RootDir, DirectoryContent::IncludeDirsFlag, DirContent);
+ GetDirectoryContent(m_RootDir, DirectoryContentFlags::IncludeDirs, DirContent);
// Initialize buckets
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index e6b6be525..cca51e63e 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -10,12 +10,13 @@
#include <zencore/scopeguard.h>
#include <zencore/timer.h>
#include <zencore/trace.h>
+#include <zenhttp/packageformat.h>
#include <zenstore/cache/cacheshared.h>
#include <zenstore/cache/structuredcachestore.h>
#include <zenstore/cache/upstreamcacheclient.h>
#include <zenstore/cidstore.h>
#include <zenutil/cache/cacherequests.h>
-#include <zenutil/packageformat.h>
+#include <zenutil/workerpools.h>
#include <zencore/memory/llm.h>
@@ -561,7 +562,17 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
Request.RecordObject = CbObjectView(Request.RecordCacheValue.GetData());
ParseValues(Request);
- Request.Complete = true;
+ Request.Complete = true;
+ size_t ValueCount = Request.Values.size();
+ std::vector<IoHash> CidHashes;
+ std::vector<size_t> RequestValueIndexes;
+ const bool DoBatch = ValueCount > 7;
+ if (DoBatch)
+ {
+ CidHashes.reserve(ValueCount);
+ RequestValueIndexes.reserve(ValueCount);
+ }
+ size_t ValueIndex = 0;
for (ValueRequestData& Value : Request.Values)
{
CachePolicy ValuePolicy = Value.DownstreamPolicy;
@@ -596,6 +607,11 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
Request.Complete = false;
}
}
+ else if (DoBatch)
+ {
+ CidHashes.push_back(Value.ContentId);
+ RequestValueIndexes.push_back(ValueIndex);
+ }
else
{
if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Value.ContentId))
@@ -611,7 +627,6 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
ZEN_WARN("Skipping invalid chunk in local cache '{}'", Value.ContentId);
}
}
-
if (EnumHasAllFlags(ValuePolicy, CachePolicy::QueryRemote))
{
NeedUpstreamAttachment = true;
@@ -619,6 +634,53 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
Request.Complete = false;
}
+ ValueIndex++;
+ }
+ if (!RequestValueIndexes.empty())
+ {
+ m_CidStore.IterateChunks(
+ CidHashes,
+ [this, &Request, ValueCount, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool {
+ try
+ {
+ const size_t ValueIndex = RequestValueIndexes[Index];
+ ValueRequestData& Value = Request.Values[ValueIndex];
+ if (Payload)
+ {
+ Value.Payload = CompressedBuffer::FromCompressedNoValidate(IoBuffer(Payload)).MakeOwned();
+ if (Value.Payload)
+ {
+ Value.Exists = true;
+ }
+ else
+ {
+ ZEN_WARN("Skipping invalid chunk in local cache '{}'", Value.ContentId);
+ }
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("CacheRpcHandler::HandleRpcGetCacheRecords IterateChunks callback failed with '{}'", Ex.what());
+ }
+ return true;
+ },
+ &GetSmallWorkerPool(EWorkloadType::Burst),
+ 64u * 1024u);
+
+ for (size_t Index : RequestValueIndexes)
+ {
+ ValueRequestData& Value = Request.Values[Index];
+ if (!Value.Exists)
+ {
+ const CachePolicy ValuePolicy = Value.DownstreamPolicy;
+ if (EnumHasAllFlags(ValuePolicy, CachePolicy::QueryRemote))
+ {
+ NeedUpstreamAttachment = true;
+ Value.ReadFromUpstream = true;
+ }
+ Request.Complete = false;
+ }
+ }
}
}
}
@@ -1415,6 +1477,7 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
using namespace cache::detail;
const bool HasUpstream = m_UpstreamCache.IsActive();
+ // TODO: BatchGet records?
std::vector<CacheKeyRequest*> UpstreamRecordRequests;
for (size_t RecordIndex = 0; RecordIndex < Records.size(); ++RecordIndex)
{
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index c14ea73a8..133cb42d7 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -424,7 +424,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc,
ZEN_INFO("initializing cache store at '{}'", m_BasePath);
DirectoryContent DirContent;
- GetDirectoryContent(m_BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
+ GetDirectoryContent(m_BasePath, DirectoryContentFlags::IncludeDirs, DirContent);
std::vector<std::string> Namespaces;
for (const std::filesystem::path& DirPath : DirContent.Directories)
diff --git a/src/zenstore/chunkedfile.cpp b/src/zenstore/chunkedfile.cpp
index 0b66c7b9b..f200bc1ec 100644
--- a/src/zenstore/chunkedfile.cpp
+++ b/src/zenstore/chunkedfile.cpp
@@ -1,7 +1,7 @@
// Copyright Epic Games, Inc. All Rights Reserved.
+#include <zencore/basicfile.h>
#include <zenstore/chunkedfile.h>
-#include <zenutil/basicfile.h>
#include "chunking.h"
@@ -195,10 +195,10 @@ TEST_CASE("chunkedfile.findparams")
{
# if 1
DirectoryContent SourceContent1;
- GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContent::IncludeFilesFlag, SourceContent1);
+ GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContentFlags::IncludeFiles, SourceContent1);
const std::vector<std::filesystem::path>& SourceFiles1 = SourceContent1.Files;
DirectoryContent SourceContent2;
- GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContent::IncludeFilesFlag, SourceContent2);
+ GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContentFlags::IncludeFiles, SourceContent2);
const std::vector<std::filesystem::path>& SourceFiles2 = SourceContent2.Files;
# else
std::filesystem::path SourcePath1 =
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 50af7246e..2be0542db 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -199,6 +199,7 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const
// reads, insert and GC.
m_BlockStore.WriteChunk(ChunkData, ChunkSize, m_PayloadAlignment, [&](const BlockStoreLocation& Location) {
+ ZEN_MEMSCOPE(GetCasContainerTag());
ZEN_TRACE_CPU("CasContainer::UpdateLocation");
BlockStoreDiskLocation DiskLocation(Location, m_PayloadAlignment);
const CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = DiskLocation};
@@ -232,7 +233,6 @@ CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash>
ZEN_ASSERT(Chunks.size() == ChunkHashes.size());
std::vector<CasStore::InsertResult> Result(Chunks.size());
std::vector<size_t> NewChunkIndexes;
- Result.reserve(Chunks.size());
{
RwLock::SharedLockScope _(m_LocationMapLock);
for (size_t ChunkIndex = 0; ChunkIndex < ChunkHashes.size(); ChunkIndex++)
@@ -264,6 +264,7 @@ CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash>
size_t ChunkOffset = 0;
m_BlockStore.WriteChunks(Datas, m_PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) {
+ ZEN_MEMSCOPE(GetCasContainerTag());
std::vector<CasDiskIndexEntry> IndexEntries;
for (const BlockStoreLocation& Location : Locations)
{
@@ -345,12 +346,13 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes,
}
}
}
- if (FoundChunkLocations.size() < 3)
+ if (FoundChunkLocations.size() < 4)
{
- for (size_t ChunkIndex : FoundChunkIndexes)
+ for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++)
{
- IoBuffer Chunk = m_BlockStore.TryGetChunk(FoundChunkLocations[ChunkIndex]);
- if (!AsyncCallback(ChunkIndex, Chunk))
+ IoBuffer Chunk = m_BlockStore.TryGetChunk(FoundChunkLocations[Index]);
+ size_t OuterIndex = FoundChunkIndexes[Index];
+ if (!AsyncCallback(OuterIndex, Chunk))
{
return false;
}
@@ -359,6 +361,18 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes,
}
auto DoOneBlock = [&](std::span<const size_t> ChunkIndexes) {
+ if (ChunkIndexes.size() < 4)
+ {
+ for (size_t ChunkIndex : ChunkIndexes)
+ {
+ IoBuffer Chunk = m_BlockStore.TryGetChunk(FoundChunkLocations[ChunkIndex]);
+ if (!AsyncCallback(FoundChunkIndexes[ChunkIndex], Chunk))
+ {
+ return false;
+ }
+ }
+ return true;
+ }
return m_BlockStore.IterateBlock(
FoundChunkLocations,
ChunkIndexes,
@@ -378,19 +392,7 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes,
Latch WorkLatch(1);
std::atomic_bool AsyncContinue = true;
bool Continue = m_BlockStore.IterateChunks(FoundChunkLocations, [&](uint32_t BlockIndex, std::span<const size_t> ChunkIndexes) {
- if (ChunkIndexes.size() < 3)
- {
- for (size_t ChunkIndex : ChunkIndexes)
- {
- IoBuffer Chunk = m_BlockStore.TryGetChunk(FoundChunkLocations[ChunkIndex]);
- if (!AsyncCallback(FoundChunkIndexes[ChunkIndex], Chunk))
- {
- return false;
- }
- }
- return true;
- }
- else if (OptionalWorkerPool)
+ if (OptionalWorkerPool && (ChunkIndexes.size() > 3))
{
WorkLatch.AddCount(1);
OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() {
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 82dbe3551..14123528c 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -2,6 +2,7 @@
#include "filecas.h"
+#include <zencore/basicfile.h>
#include <zencore/compress.h>
#include <zencore/except.h>
#include <zencore/filesystem.h>
@@ -19,7 +20,6 @@
#include <zencore/workthreadpool.h>
#include <zenstore/gc.h>
#include <zenstore/scrubcontext.h>
-#include <zenutil/basicfile.h>
#if ZEN_WITH_TESTS
# include <zencore/compactbinarybuilder.h>
@@ -185,7 +185,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
// in this folder as well
struct Visitor : public FileSystemTraversal::TreeVisitor
{
- virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override
+ virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t, uint32_t) override
{
// We don't care about files
}
@@ -193,8 +193,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
{
return std::find(&HexChars[0], &HexChars[16], C) != &HexChars[16];
}
- virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
- [[maybe_unused]] const path_view& DirectoryName) override
+ virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName, uint32_t) override
{
if (DirectoryName.length() == 3)
{
@@ -1175,7 +1174,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir)
struct Visitor : public FileSystemTraversal::TreeVisitor
{
Visitor(const std::filesystem::path& RootDir, std::vector<FileCasIndexEntry>& Entries) : RootDirectory(RootDir), Entries(Entries) {}
- virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
{
std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory);
@@ -1201,11 +1200,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir)
}
}
- virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
- [[maybe_unused]] const path_view& DirectoryName) override
- {
- return true;
- }
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
const std::filesystem::path& RootDirectory;
std::vector<FileCasIndexEntry>& Entries;
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 50588b8c0..7ac10d613 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1810,7 +1810,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons
}
else
{
- ZEN_ERROR("writing gc result failed with system error exception: '{}'", SystemError.what());
+ ZEN_ERROR("writing gc result failed with system error exception: '{}' ({})", SystemError.what(), SystemError.code().value());
}
}
catch (const std::bad_alloc& BadAlloc)
@@ -2274,7 +2274,9 @@ GcScheduler::SchedulerThread()
}
else
{
- ZEN_ERROR("scheduling garbage collection failed with system error exception: '{}'", SystemError.what());
+ ZEN_ERROR("scheduling garbage collection failed with system error exception: '{}' ({})",
+ SystemError.what(),
+ SystemError.code().value());
}
m_LastGcTime = GcClock::Now();
m_LastLightweightGcTime = m_LastGcTime;
@@ -2596,11 +2598,15 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
}
if (RetryCount == 0)
{
- ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}'", SystemError.what());
+ ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}' ({})",
+ SystemError.what(),
+ SystemError.code().value());
}
else
{
- ZEN_WARN("writing gc scheduler state failed with system error exception: '{}'", SystemError.what());
+ ZEN_WARN("writing gc scheduler state failed with system error exception: '{}' ({})",
+ SystemError.what(),
+ SystemError.code().value());
}
}
catch (const std::bad_alloc& BadAlloc)
@@ -2636,11 +2642,15 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
{
if (SilenceErrors)
{
- ZEN_WARN("scheduling garbage collection failed with system error exception: '{}'", SystemError.what());
+ ZEN_WARN("scheduling garbage collection failed with system error exception: '{}' ({})",
+ SystemError.what(),
+ SystemError.code().value());
}
else
{
- ZEN_ERROR("scheduling garbage collection failed with system error exception: '{}'", SystemError.what());
+ ZEN_ERROR("scheduling garbage collection failed with system error exception: '{}' ({})",
+ SystemError.what(),
+ SystemError.code().value());
}
}
m_LastGcTime = GcClock::Now();
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index 8f8f2ccd7..97357e5cb 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -2,9 +2,9 @@
#pragma once
+#include <zencore/basicfile.h>
#include <zencore/filesystem.h>
#include <zencore/zencore.h>
-#include <zenutil/basicfile.h>
ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
diff --git a/src/zenstore/include/zenstore/caslog.h b/src/zenstore/include/zenstore/caslog.h
index edb4f8d9b..3d95c9c90 100644
--- a/src/zenstore/include/zenstore/caslog.h
+++ b/src/zenstore/include/zenstore/caslog.h
@@ -2,8 +2,8 @@
#pragma once
+#include <zencore/basicfile.h>
#include <zencore/uid.h>
-#include <zenutil/basicfile.h>
namespace zen {
diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp
index 80e03296c..02a83d2a6 100644
--- a/src/zenstore/workspaces.cpp
+++ b/src/zenstore/workspaces.cpp
@@ -2,13 +2,13 @@
#include "zenstore/workspaces.h"
+#include <zencore/basicfile.h>
#include <zencore/compactbinarybuilder.h>
#include <zencore/fmtutils.h>
#include <zencore/scopeguard.h>
#include <zencore/timer.h>
#include <zencore/trace.h>
#include <zencore/workthreadpool.h>
-#include <zenutil/basicfile.h>
ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_set.h>
@@ -252,18 +252,43 @@ FolderStructure::FolderStructure(std::vector<FileEntry>&& InEntries, std::vector
}
namespace {
- struct FolderScanner
+ struct FolderScanner : public GetDirectoryContentVisitor
{
FolderScanner(LoggerRef& Log, WorkerThreadPool& WorkerPool, const std::filesystem::path& Path)
: m_Log(Log)
, Path(Path)
- , WorkLatch(1)
, WorkerPool(WorkerPool)
{
}
void Traverse();
- void Traverse(const std::filesystem::path& RelativeRoot, const std::filesystem::path& Path);
+
+ virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
+ {
+ std::vector<FolderStructure::FileEntry> FileEntries;
+ std::vector<Oid> PathIds;
+ const size_t FileCount = Content.FileNames.size();
+ FileEntries.reserve(FileCount);
+ PathIds.reserve(FileCount);
+
+ auto FileNameIt = Content.FileNames.begin();
+ auto FileSizeIt = Content.FileSizes.begin();
+ while (FileNameIt != Content.FileNames.end())
+ {
+ ZEN_ASSERT_SLOW(FileSizeIt != Content.FileSizes.end());
+
+ std::filesystem::path RelativePath = RelativeRoot.empty() ? *FileNameIt : RelativeRoot / *FileNameIt;
+ PathIds.emplace_back(Workspaces::PathToId(RelativePath));
+ FileEntries.emplace_back(FolderStructure::FileEntry{.RelativePath = std::move(RelativePath), .Size = *FileSizeIt});
+
+ FileNameIt++;
+ FileSizeIt++;
+ }
+ WorkLock.WithExclusiveLock([&]() {
+ FoundFiles.insert(FoundFiles.end(), FileEntries.begin(), FileEntries.end());
+ FoundFileIds.insert(FoundFileIds.end(), PathIds.begin(), PathIds.end());
+ });
+ }
LoggerRef& Log() { return m_Log; }
LoggerRef& m_Log;
@@ -271,75 +296,34 @@ namespace {
RwLock WorkLock;
std::vector<FolderStructure::FileEntry> FoundFiles;
std::vector<Oid> FoundFileIds;
- Latch WorkLatch;
WorkerThreadPool& WorkerPool;
};
- struct Visitor : public FileSystemTraversal::TreeVisitor
+ void FolderScanner::Traverse()
{
- Visitor(FolderScanner& Data, const std::filesystem::path& RelativeRoot) : Data(Data), RelativeRoot(RelativeRoot) {}
+ Stopwatch Timer;
- FileSystemTraversal Traverser;
- FolderScanner& Data;
- std::vector<FolderStructure::FileEntry> Entries;
- std::vector<Oid> FileIds;
- std::filesystem::path RelativeRoot;
+ const std::filesystem::path Root = std::filesystem::absolute(Path);
- virtual void VisitFile(const std::filesystem::path&, const path_view& File, uint64_t FileSize)
- {
- std::filesystem::path RelativePath = RelativeRoot.empty() ? File : RelativeRoot / File;
- Entries.push_back(FolderStructure::FileEntry{.RelativePath = RelativePath, .Size = FileSize});
- FileIds.push_back(Workspaces::PathToId(RelativePath));
- }
+ Latch WorkLatch(1);
- virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName)
- {
- ZEN_ASSERT(!Parent.empty());
- ZEN_ASSERT(!DirectoryName.empty());
- FolderScanner* DataPtr = &Data;
- Data.WorkLatch.AddCount(1);
- Data.WorkerPool.ScheduleWork([DataPtr,
- RootDir = Parent / DirectoryName,
- RelativeRoot = RelativeRoot.empty() ? DirectoryName : RelativeRoot / DirectoryName]() {
- auto _ = MakeGuard([DataPtr]() { DataPtr->WorkLatch.CountDown(); });
- try
- {
- DataPtr->Traverse(RelativeRoot, RootDir);
- }
- catch (const std::exception& Ex)
- {
- ZEN_WARN("Exception while traversing path {} {}: {}", RelativeRoot, RootDir, Ex.what());
- }
- });
- return false;
- }
- };
+ GetDirectoryContent(
+ Root,
+ DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes | DirectoryContentFlags::Recursive,
+ *this,
+ WorkerPool,
+ WorkLatch);
- void FolderScanner::Traverse()
- {
- Stopwatch Timer;
- Traverse({}, std::filesystem::absolute(Path));
WorkLatch.CountDown();
while (!WorkLatch.Wait(1000))
{
- WorkLock.WithSharedLock([&]() { ZEN_INFO("Found {} files in '{}'...", FoundFiles.size(), Path.string()); });
+ WorkLock.WithSharedLock([&]() { ZEN_INFO("Found {} files in '{}'...", FoundFiles.size(), Root.string()); });
}
+
ZEN_ASSERT(FoundFiles.size() == FoundFileIds.size());
ZEN_INFO("Found {} files in '{}' in {}", FoundFiles.size(), Path.string(), NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
}
- void FolderScanner::Traverse(const std::filesystem::path& RelativeRoot, const std::filesystem::path& AbsoluteRoot)
- {
- Visitor LeafVisitor(*this, RelativeRoot);
- LeafVisitor.Traverser.TraverseFileSystem(AbsoluteRoot, LeafVisitor);
- if (!LeafVisitor.Entries.empty())
- {
- WorkLock.WithExclusiveLock([&]() {
- FoundFiles.insert(FoundFiles.end(), LeafVisitor.Entries.begin(), LeafVisitor.Entries.end());
- FoundFileIds.insert(FoundFileIds.end(), LeafVisitor.FileIds.begin(), LeafVisitor.FileIds.end());
- });
- }
- }
} // namespace
std::unique_ptr<FolderStructure>
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-11 11:13:10 +0000