diff options
| author | Dan Engelbrecht <[email protected]> | 2025-03-31 10:24:39 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-03-31 10:24:39 +0200 |
| commit | fd2efb5af872a357dbc0f729f4101a330dcb4fda (patch) | |
| tree | 949e933156467113b861f4b0ca5862f9cdf10189 /src | |
| parent | check file from local track state during download (#329) (diff) | |
| download | zen-fd2efb5af872a357dbc0f729f4101a330dcb4fda.tar.xz zen-fd2efb5af872a357dbc0f729f4101a330dcb4fda.zip | |
long filename support (#330)
- Bugfix: Long file paths now works correctly on Windows
Diffstat (limited to 'src')
41 files changed, 1270 insertions, 519 deletions
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 835e01151..573639c2d 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -688,7 +688,7 @@ Copy(const std::filesystem::path& Source, const std::filesystem::path& Target) static bool TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target) { - if (!std::filesystem::is_regular_file(Source)) + if (!IsFile(Source)) { return false; } @@ -717,7 +717,7 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::filesystem::path DataPath = StringToPath(m_DataPath); std::filesystem::path TargetPath = StringToPath(m_TargetPath); - if (!std::filesystem::is_directory(DataPath)) + if (!IsDir(DataPath)) { throw OptionParseException("data path must exist"); } diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index e3dc20621..d2ba20e78 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -165,12 +165,11 @@ namespace { std::filesystem::path MakeSafeAbsolutePath(const std::string Path) { std::filesystem::path AbsolutePath = std::filesystem::absolute(StringToPath(Path)).make_preferred(); -#if ZEN_PLATFORM_WINDOWS && \ - 0 // TODO: We need UNC for long file names but we need to stop using std::filesystem for those paths - std::filesystem::* functions +#if ZEN_PLATFORM_WINDOWS && 1 const std::string_view Prefix = "\\\\?\\"; const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); std::u8string PathString = AbsolutePath.u8string(); - if (!PathString.starts_with(PrefixU8)) + if (!PathString.empty() && !PathString.starts_with(PrefixU8)) { PathString.insert(0, PrefixU8); return std::filesystem::path(PathString); @@ -179,6 +178,82 @@ namespace { return AbsolutePath; } + void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) + { + std::error_code Ec; + RenameFile(SourcePath, TargetPath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + RenameFile(SourcePath, TargetPath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) + { + std::error_code Ec; + bool Result = SetFileReadOnly(Path, ReadOnly, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFile(Path)) + { + return false; + } + Ec.clear(); + Result = SetFileReadOnly(Path, ReadOnly, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + + void RemoveFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFile(Path)) + { + return; + } + Ec.clear(); + RemoveFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + void RemoveDirWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveDir(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsDir(Path)) + { + return; + } + Ec.clear(); + RemoveDir(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) { #if ZEN_PLATFORM_WINDOWS @@ -250,21 +325,8 @@ namespace { { try { - std::error_code Ec; - std::filesystem::remove(LocalFilePath, Ec); - if (Ec) - { - // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time - Ec.clear(); - if (std::filesystem::exists(LocalFilePath, Ec) || Ec) - { - Sleep(200); - if (std::filesystem::exists(LocalFilePath)) - { - std::filesystem::remove(LocalFilePath); - } - } - } + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); } catch (const std::exception& Ex) { @@ -273,7 +335,7 @@ namespace { } } - for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) { bool Leave = false; for (const std::string_view ExcludeDirectory : ExcludeDirectories) @@ -288,23 +350,27 @@ namespace { { try { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); - } - catch (const std::exception&) - { - Sleep(200); - try + std::error_code Ec; + zen::CleanDirectory(LocalDirPath, true, Ec); + if (Ec) { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); + Sleep(200); + zen::CleanDirectory(LocalDirPath, true); + Ec.clear(); } - catch (const std::exception& Ex) + + RemoveDir(LocalDirPath, Ec); + if (Ec) { - ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); - CleanWipe = false; + Sleep(200); + RemoveDir(LocalDirPath); } } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); + CleanWipe = false; + } } } return CleanWipe; @@ -312,7 +378,7 @@ namespace { std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) { - if (!std::filesystem::is_regular_file(Path)) + if (!IsFile(Path)) { throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); } @@ -355,7 +421,7 @@ namespace { const IoHash& Hash, const std::string& Suffix = {}) { - std::filesystem::path TempFilePath = (TempFolderPath / (Hash.ToHexString() + Suffix)).make_preferred(); + std::filesystem::path TempFilePath = TempFolderPath / (Hash.ToHexString() + Suffix); return WriteToTempFile(std::move(Buffer), TempFilePath); } @@ -453,12 +519,12 @@ namespace { std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) { - return (CacheFolderPath / (RawHash.ToHexString() + ".tmp")).make_preferred(); + return CacheFolderPath / (RawHash.ToHexString() + ".tmp"); } std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) { - return (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + return CacheFolderPath / RawHash.ToHexString(); } ChunkedFolderContent ScanAndChunkFolder( @@ -1611,7 +1677,8 @@ namespace { auto __ = MakeGuard([&TempFolder]() { if (CleanDirectory(TempFolder, {})) { - std::filesystem::remove(TempFolder); + std::error_code DummyEc; + RemoveDir(TempFolder, DummyEc); } }); @@ -1888,7 +1955,7 @@ namespace { } ZEN_ASSERT_SLOW(IoHash::HashBuffer(RawSource) == ChunkHash); { - std::filesystem::path TempFilePath = (TempFolderPath / ChunkHash.ToHexString()).make_preferred(); + std::filesystem::path TempFilePath = TempFolderPath / ChunkHash.ToHexString(); BasicFile CompressedFile; std::error_code Ec; @@ -1930,7 +1997,7 @@ namespace { return Compressed.GetCompressed(); } CompressedFile.Close(); - std::filesystem::remove(TempFilePath, Ec); + RemoveFile(TempFilePath, Ec); ZEN_UNUSED(Ec); } @@ -2715,7 +2782,8 @@ namespace { auto _ = MakeGuard([&]() { if (CleanDirectory(ZenTempFolder, {})) { - std::filesystem::remove(ZenTempFolder); + std::error_code DummyEc; + RemoveDir(ZenTempFolder, DummyEc); } }); CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); @@ -2861,7 +2929,7 @@ namespace { { std::filesystem::path ExcludeManifestPath = Path / ZenExcludeManifestName; tsl::robin_set<std::string> ExcludeAssetPaths; - if (std::filesystem::is_regular_file(ExcludeManifestPath)) + if (IsFile(ExcludeManifestPath)) { std::vector<std::filesystem::path> AssetPaths = ParseManifest(Path, ExcludeManifestPath); ExcludeAssetPaths.reserve(AssetPaths.size()); @@ -2904,12 +2972,13 @@ namespace { for (const std::filesystem::path& AssetPath : AssetPaths) { Content.Paths.push_back(AssetPath); - Content.RawSizes.push_back(std::filesystem::file_size(Path / AssetPath)); + const std::filesystem::path AssetFilePath = (Path / AssetPath).make_preferred(); + Content.RawSizes.push_back(FileSizeFromPath(AssetFilePath)); #if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributes(Path / AssetPath)); + Content.Attributes.push_back(GetFileAttributes(AssetFilePath)); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(Path / AssetPath)); + Content.Attributes.push_back(GetFileMode(AssetFilePath)); #endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); LocalFolderScanStats.AcceptedFileCount++; @@ -2917,12 +2986,13 @@ namespace { if (ManifestPath.is_relative()) { Content.Paths.push_back(ManifestPath); - Content.RawSizes.push_back(std::filesystem::file_size(ManifestPath)); + const std::filesystem::path ManifestFilePath = (Path / ManifestPath).make_preferred(); + Content.RawSizes.push_back(FileSizeFromPath(ManifestFilePath)); #if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributes(ManifestPath)); + Content.Attributes.push_back(GetFileAttributes(ManifestFilePath)); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(ManifestPath)); + Content.Attributes.push_back(GetFileMode(ManifestFilePath)); #endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); @@ -3751,7 +3821,7 @@ namespace { if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) { const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!std::filesystem::exists(TargetPath)) + if (!IsFile(TargetPath)) { ErrorLock.WithExclusiveLock([&]() { Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); @@ -3761,7 +3831,7 @@ namespace { else { std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow<uint64_t>(std::filesystem::file_size(TargetPath, Ec)); + uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec)); if (Ec) { ErrorLock.WithExclusiveLock([&]() { @@ -3997,9 +4067,15 @@ namespace { void FinalizeChunkSequence(const std::filesystem::path& TargetFolder, const IoHash& SequenceRawHash) { ZEN_TRACE_CPU("FinalizeChunkSequence"); - ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - std::filesystem::rename(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), - GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + std::error_code Ec; + RenameFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), + GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash), + Ec); + if (Ec) + { + throw std::system_error(Ec); + } } void FinalizeChunkSequences(const std::filesystem::path& TargetFolder, @@ -4646,7 +4722,7 @@ namespace { Payload.SetDeleteOnClose(false); Payload = {}; CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); - std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); + RenameFile(TempBlobPath, CompressedChunkPath, Ec); if (Ec) { CompressedChunkPath = std::filesystem::path{}; @@ -4717,7 +4793,7 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - std::filesystem::remove(CompressedChunkPath); + RemoveFile(CompressedChunkPath); std::vector<uint32_t> CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -4823,13 +4899,13 @@ namespace { const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); continue; } } } - std::filesystem::remove(CacheDirContent.Files[Index]); + RemoveFileWithRetry(CacheDirContent.Files[Index]); } } @@ -4875,7 +4951,7 @@ namespace { } } } - std::filesystem::remove(BlockDirContent.Files[Index]); + RemoveFileWithRetry(BlockDirContent.Files[Index]); } } @@ -4896,7 +4972,7 @@ namespace { // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); } else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) @@ -4905,14 +4981,14 @@ namespace { // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); } else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) { const uint32_t LocalSequenceIndex = It->second; const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); CacheMappingStats.LocalPathsMatchingSequencesCount++; @@ -5181,7 +5257,7 @@ namespace { std::filesystem::path BlockPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - if (std::filesystem::exists(BlockPath)) + if (IsFile(BlockPath)) { CachedChunkBlockIndexes.push_back(BlockIndex); UsingCachedBlock = true; @@ -5462,7 +5538,7 @@ namespace { const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; std::filesystem::path CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); - if (std::filesystem::exists(CompressedChunkPath)) + if (IsFile(CompressedChunkPath)) { IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); if (ExistingCompressedPart) @@ -5481,7 +5557,7 @@ namespace { else { std::error_code DummyEc; - std::filesystem::remove(CompressedChunkPath, DummyEc); + RemoveFile(CompressedChunkPath, DummyEc); } } } @@ -5543,7 +5619,7 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - std::filesystem::remove(CompressedChunkPath); + RemoveFile(CompressedChunkPath); std::vector<uint32_t> CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -5923,11 +5999,11 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } WritePartsComplete++; - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -6008,7 +6084,7 @@ namespace { BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { BlockChunkPath = std::filesystem::path{}; @@ -6079,14 +6155,14 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error( fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); } if (!BlockChunkPath.empty()) { - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); } WritePartsComplete++; @@ -6180,7 +6256,7 @@ namespace { BlockBuffer = {}; BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { BlockChunkPath = std::filesystem::path{}; @@ -6259,14 +6335,14 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error( fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } if (!BlockChunkPath.empty()) { - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); } WritePartsComplete++; @@ -6398,7 +6474,7 @@ namespace { const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); if (!WipeTargetFolder) { @@ -6430,27 +6506,18 @@ namespace { if (!CachedRemoteSequences.contains(RawHash)) { // We need it - ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); - std::error_code Ec; - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) - { - Sleep(100); - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); - } - if (Ec) - { - zen::ThrowSystemError(Ec.value(), Ec.message()); - } + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedRemoteSequences.insert(RawHash); CachedCount++; } else { // We already have it - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); SkippedCount++; } } @@ -6499,8 +6566,8 @@ namespace { for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) { const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - SetFileReadOnly(LocalFilePath, false); - std::filesystem::remove(LocalFilePath); + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); } } } @@ -6579,9 +6646,9 @@ namespace { std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) { - if (std::filesystem::exists(TargetFilePath)) + if (IsFile(TargetFilePath)) { - SetFileReadOnly(TargetFilePath, false); + SetFileReadOnlyWithRetry(TargetFilePath, false); } else { @@ -6615,13 +6682,13 @@ namespace { if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); } else { - if (std::filesystem::exists(FirstTargetFilePath)) + if (IsFile(FirstTargetFilePath)) { - SetFileReadOnly(FirstTargetFilePath, false); + SetFileReadOnlyWithRetry(FirstTargetFilePath, false); } else { @@ -6634,7 +6701,7 @@ namespace { const uint32_t LocalPathIndex = InplaceIt->second; const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); - ZEN_ASSERT_SLOW(std::filesystem::exists(SourceFilePath)); + ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; @@ -6643,19 +6710,10 @@ namespace { { const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + + RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); - std::error_code Ec; - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) - { - Sleep(100); - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); - } - if (Ec) - { - zen::ThrowSystemError(Ec.value(), Ec.message()); - } RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; } } @@ -6685,20 +6743,20 @@ namespace { if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_ASSERT_SLOW(std::filesystem::exists(TargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(TargetFilePath)); } else { - if (std::filesystem::exists(TargetFilePath)) + if (IsFile(TargetFilePath)) { - SetFileReadOnly(TargetFilePath, false); + SetFileReadOnlyWithRetry(TargetFilePath, false); } else { CreateDirectories(TargetFilePath.parent_path()); } - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } @@ -7167,7 +7225,7 @@ namespace { ChunkedFolderContent LocalContent; bool HasLocalState = false; - if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) + if (IsFile(ZenStateFilePath(ZenFolderPath))) { try { @@ -7268,9 +7326,9 @@ namespace { { const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); - if (std::filesystem::exists(LocalFilePath)) + if (IsFile(LocalFilePath)) { - const uint64_t FileSize = std::filesystem::file_size(LocalFilePath); + const uint64_t FileSize = FileSizeFromPath(LocalFilePath); OutLocalFolderContent.Paths[PathRangeIndex] = FilePath; OutLocalFolderContent.RawSizes[PathRangeIndex] = FileSize; OutLocalFolderContent.Attributes[PathRangeIndex] = GetNativeFileAttributes(LocalFilePath); @@ -7502,7 +7560,7 @@ namespace { FolderContent LocalFolderContent; if (!PrimeCacheOnly) { - if (std::filesystem::is_directory(Path)) + if (IsDir(Path)) { if (!WipeTargetFolder) { @@ -7692,7 +7750,7 @@ namespace { } if (CleanDirectory(ZenTempFolder, {})) { - std::filesystem::remove(ZenTempFolder); + RemoveDirWithRetry(ZenTempFolder); } } @@ -8479,7 +8537,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -8592,7 +8650,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -8721,7 +8779,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageCache::Statistics StorageCacheStats; const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -8799,7 +8857,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageCache::Statistics StorageCacheStats; const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -8871,7 +8929,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (DownloadPath / ZenFolderName) : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? DownloadPath / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -9000,10 +9058,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { Work.ScheduleWork( GetIOWorkerPool(), - [SourceSize, FilePath](std::atomic<bool>&) { + [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { if (!AbortFlag) { - bool IsReadOnly = SetFileReadOnly(FilePath, false); + bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false); { BasicFile Source(FilePath, BasicFile::Mode::kWrite); uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); @@ -9030,7 +9088,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } break; case 1: - std::filesystem::remove(FilePath); + { + (void)SetFileReadOnlyWithRetry(FilePath, false); + RemoveFileWithRetry(FilePath); + } break; default: break; @@ -9184,7 +9245,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -9240,7 +9301,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index 6ec6a80db..185edc35d 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -629,7 +629,7 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_OutputPath.empty()) { TargetPath = std::filesystem::path(m_OutputPath); - if (std::filesystem::is_directory(TargetPath)) + if (IsDir(TargetPath)) { TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp index cc6ddd505..53e80c896 100644 --- a/src/zen/cmds/copy_cmd.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -64,8 +64,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource); - const bool IsDirCopy = std::filesystem::is_directory(m_CopySource); + const bool IsFileCopy = IsFile(m_CopySource); + const bool IsDirCopy = IsDir(m_CopySource); if (!IsFileCopy && !IsDirCopy) { @@ -79,20 +79,14 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (IsDirCopy) { - if (std::filesystem::exists(ToPath)) + if (IsFile(ToPath)) { - const bool IsTargetDir = std::filesystem::is_directory(ToPath); - if (!IsTargetDir) - { - if (std::filesystem::is_regular_file(ToPath)) - { - throw std::runtime_error("Attempted copy of directory into file"); - } - } + throw std::runtime_error("Attempted copy of directory into file"); } - else + + if (!IsDir(ToPath)) { - std::filesystem::create_directories(ToPath); + CreateDirectories(ToPath); } std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec); diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 6bc499f03..13c7c4b23 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -41,7 +41,7 @@ namespace { std::string ReadJupiterAccessTokenFromFile(const std::filesystem::path& Path) { - if (!std::filesystem::is_regular_file(Path)) + if (!IsFile(Path)) { throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); } @@ -2185,7 +2185,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return 1; } - std::filesystem::remove_all(TmpPath); + DeleteDirectories(TmpPath); ZEN_CONSOLE("mirrored {} files from {} oplog entries successfully", FileCount.load(), OplogEntryCount); diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp index 5b88a1f73..4fc38d92a 100644 --- a/src/zen/cmds/rpcreplay_cmd.cpp +++ b/src/zen/cmds/rpcreplay_cmd.cpp @@ -196,7 +196,7 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException("Rpc replay command requires a path"); } - if (!std::filesystem::exists(m_RecordingPath) || !std::filesystem::is_directory(m_RecordingPath)) + if (!IsDir(m_RecordingPath)) { throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath)); } diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp index a99ba9704..309b8996a 100644 --- a/src/zen/cmds/run_cmd.cpp +++ b/src/zen/cmds/run_cmd.cpp @@ -100,7 +100,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - CleanDirectory(BaseDirectory); + CleanDirectory(BaseDirectory, /*ForceRemoveReadOnlyFiles*/ false); } } diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp index f87725e36..64039e4c9 100644 --- a/src/zen/cmds/serve_cmd.cpp +++ b/src/zen/cmds/serve_cmd.cpp @@ -67,7 +67,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException("command requires a root path"); } - if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath)) + if (!IsDir(m_RootPath)) { throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath)); } diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index 4d1534e05..88c0b22a2 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -33,7 +33,7 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_DataDir.empty()) { std::filesystem::path DataDir = StringToPath(m_DataDir); - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index 44a41146c..aacc115a0 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -159,7 +159,7 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; @@ -229,7 +229,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index a181bbd66..ea526399c 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -590,7 +590,7 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std:: // deleting the temporary file BasicFile::Close(); - std::filesystem::rename(m_TempPath, FinalFileName, Ec); + RenameFile(m_TempPath, FinalFileName, Ec); if (Ec) { @@ -984,9 +984,9 @@ TEST_CASE("TemporaryFile") TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); Path = TmpFile.GetPath(); - CHECK(std::filesystem::exists(Path)); + CHECK(IsFile(Path)); } - CHECK(std::filesystem::exists(Path) == false); + CHECK(IsFile(Path) == false); } SUBCASE("MoveIntoPlace") @@ -997,11 +997,11 @@ TEST_CASE("TemporaryFile") CHECK(!Ec); std::filesystem::path TempPath = TmpFile.GetPath(); std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; - CHECK(std::filesystem::exists(TempPath)); + CHECK(IsFile(TempPath)); TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); CHECK(!Ec); - CHECK(std::filesystem::exists(TempPath) == false); - CHECK(std::filesystem::exists(FinalPath)); + CHECK(IsFile(TempPath) == false); + CHECK(IsFile(FinalPath)); } } diff --git a/src/zencore/except.cpp b/src/zencore/except.cpp index d5eabea9d..610b0ced5 100644 --- a/src/zencore/except.cpp +++ b/src/zencore/except.cpp @@ -47,7 +47,7 @@ ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string { if (HRESULT_FACILITY(hRes) == FACILITY_WIN32) { - throw std::system_error(std::error_code(hRes & 0xffff, std::system_category()), std::string(Message)); + throw std::system_error(std::error_code(HRESULT_CODE(hRes), std::system_category()), std::string(Message)); } else { diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 8a369f02e..6ff4dd053 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -86,16 +86,9 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) } bool -CreateDirectories(const wchar_t* Dir) +CreateDirectories(const wchar_t* Path) { - // This may be suboptimal, in that it appears to try and create directories - // from the root on up instead of from some directory which is known to - // be present - // - // We should implement a smarter version at some point since this can be - // pretty expensive in aggregate - - return std::filesystem::create_directories(Dir); + return CreateDirectories(std::filesystem::path(Path)); } // Erase all files and directories in a given directory, leaving an empty directory @@ -212,75 +205,326 @@ DeleteDirectoriesInternal(const wchar_t* DirPath) bool CleanDirectory(const wchar_t* DirPath, bool KeepDotFiles) { - if (std::filesystem::exists(DirPath)) + if (IsDir(DirPath)) { return WipeDirectory(DirPath, KeepDotFiles); } return CreateDirectories(DirPath); } + +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_WINDOWS +const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; +#else +const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; +#endif // ZEN_PLATFORM_WINDOWS + +const uint32_t FileModeWriteEnableFlags = 0222; + +bool +IsFileAttributeReadOnly(uint32_t FileAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; +#else + return (FileAttributes & 0x00000001) != 0; #endif // ZEN_PLATFORM_WINDOWS +} bool -CreateDirectories(const std::filesystem::path& Dir) +IsFileModeReadOnly(uint32_t FileMode) { - if (Dir.string().ends_with(":")) + return (FileMode & FileModeWriteEnableFlags) == 0; +} + +uint32_t +MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +{ + return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); +} + +uint32_t +MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) +{ + return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); +} + +bool +RemoveDirNative(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::RemoveDirectory(Path.native().c_str()); + if (!Success) { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } return false; } - while (!std::filesystem::is_directory(Dir)) + return true; +#else + return std::filesystem::remove(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::DeleteFile(Path.native().c_str()); + if (!Success) { - if (Dir.has_parent_path()) + if (ForceRemoveReadOnlyFiles) { - CreateDirectories(Dir.parent_path()); + DWORD FileAttributes = ::GetFileAttributes(Path.native().c_str()); + if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0) + { + ::SetFileAttributes(Path.native().c_str(), MakeFileAttributeReadOnly(FileAttributes, false)); + Success = ::DeleteFile(Path.native().c_str()); + } } - std::error_code ErrorCode; - std::filesystem::create_directory(Dir, ErrorCode); - if (ErrorCode) + if (!Success) { - throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string())); + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; } - return true; } - return false; + return true; +#else + if (!ForceRemoveReadOnlyFiles) + { + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + const uint32_t Mode = (uint32_t)Stat.st_mode; + if (IsFileModeReadOnly(Mode)) + { + Ec = MakeErrorCode(EACCES); + return false; + } + } + return std::filesystem::remove(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +static void +WipeDirectoryContentInternal(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ + DirectoryContent LocalDirectoryContent; + GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); + for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) + { + RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsFile(LocalFilePath)) + { + RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec); + } + } + if (Ec) + { + return; + } + } + + for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + { + WipeDirectoryContentInternal(LocalDirPath, ForceRemoveReadOnlyFiles, Ec); + if (Ec) + { + return; + } + + RemoveDirNative(LocalDirPath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsDir(LocalDirPath)) + { + RemoveDirNative(LocalDirPath, Ec); + } + } + if (Ec) + { + return; + } + } +} + +bool +CreateDirectory(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::CreateDirectory(Path.native().c_str(), nullptr); + if (!Success) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_EXISTS: + case ERROR_ALREADY_EXISTS: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return Success; +#else + return std::filesystem::create_directory(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS } bool -DeleteDirectories(const std::filesystem::path& Dir) +CreateDirectories(const std::filesystem::path& Path) { - std::error_code ErrorCode; - return std::filesystem::remove_all(Dir, ErrorCode); + std::error_code Ec; + bool Success = CreateDirectories(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to create directories for '{}'", Path.string())); + } + return Success; } bool -CleanDirectory(const std::filesystem::path& Dir) +CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec) { - if (std::filesystem::exists(Dir)) + if (Path.string().ends_with(":")) { - bool Success = true; + return false; + } + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return false; + } + if (Exists) + { + return false; + } - for (const auto& Item : std::filesystem::directory_iterator(Dir)) + if (Path.has_parent_path()) + { + bool Result = CreateDirectories(Path.parent_path(), Ec); + if (Ec) { - std::error_code ErrorCode; - const uintmax_t RemovedCount = std::filesystem::remove_all(Item, ErrorCode); - - Success = Success && !ErrorCode && RemovedCount; + return Result; } + } + return CreateDirectory(Path, Ec); +} - return Success; +bool +CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles) +{ + std::error_code Ec; + bool Result = CleanDirectory(Path, ForceRemoveReadOnlyFiles, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to clean directory for '{}'", Path.string())); } + return Result; +} - return CreateDirectories(Dir); +bool +CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return Exists; + } + if (Exists) + { + WipeDirectoryContentInternal(Path, ForceRemoveReadOnlyFiles, Ec); + return false; + } + return CreateDirectory(Path, Ec); +} + +bool +DeleteDirectories(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = DeleteDirectories(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to delete directories for '{}'", Path.string())); + } + return Result; } bool -CleanDirectoryExceptDotFiles(const std::filesystem::path& Dir) +DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec) +{ + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return Exists; + } + + if (Exists) + { + WipeDirectoryContentInternal(Path, false, Ec); + if (Ec) + { + return false; + } + bool Result = RemoveDirNative(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsDir(Path)) + { + Result = RemoveDirNative(Path, Ec); + } + } + return Result; + } + return false; +} + +bool +CleanDirectoryExceptDotFiles(const std::filesystem::path& Path) { #if ZEN_PLATFORM_WINDOWS const bool KeepDotFiles = true; - return CleanDirectory(Dir.c_str(), KeepDotFiles); + return CleanDirectory(Path.c_str(), KeepDotFiles); #else - ZEN_UNUSED(Dir); + ZEN_UNUSED(Path); ZEN_NOT_IMPLEMENTED(); #endif @@ -637,7 +881,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop { // Validate arguments - if (FromPath.empty() || !std::filesystem::is_directory(FromPath)) + if (FromPath.empty() || !IsDir(FromPath)) throw std::runtime_error("invalid CopyTree source directory specified"); if (ToPath.empty()) @@ -646,16 +890,13 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop if (Options.MustClone && !SupportsBlockRefCounting(FromPath)) throw std::runtime_error(fmt::format("cloning not possible from '{}'", FromPath)); - if (std::filesystem::exists(ToPath)) + if (IsFile(ToPath)) { - if (!std::filesystem::is_directory(ToPath)) - { - throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath)); - } + throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath)); } - else + if (!IsDir(ToPath)) { - std::filesystem::create_directories(ToPath); + CreateDirectories(ToPath); } if (Options.MustClone && !SupportsBlockRefCounting(ToPath)) @@ -811,7 +1052,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { Outfile.Close(); std::error_code DummyEc; - std::filesystem::remove(Path, DummyEc); + RemoveFile(Path, DummyEc); ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else @@ -819,7 +1060,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { close(Fd); std::error_code DummyEc; - std::filesystem::remove(Path, DummyEc); + RemoveFile(Path, DummyEc); ThrowLastError(fmt::format("File write failed for '{}'", Path)); } #endif // ZEN_PLATFORM_WINDOWS @@ -1172,7 +1413,7 @@ void FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) { #if ZEN_PLATFORM_WINDOWS - uint64_t FileInfoBuffer[8 * 1024]; + std::vector<uint64_t> FileInfoBuffer(8 * 1024); FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; bool Continue = true; @@ -1183,7 +1424,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr if (FAILED(hRes)) { - if (hRes == ERROR_FILE_NOT_FOUND || hRes == ERROR_PATH_NOT_FOUND) + if (HRESULT_CODE(hRes) == ERROR_FILE_NOT_FOUND || HRESULT_CODE(hRes) == ERROR_PATH_NOT_FOUND) { // Directory no longer exist, treat it as empty return; @@ -1193,8 +1434,9 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr while (Continue) { - BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer); - FibClass = FileIdBothDirectoryInfo; // Set up for next iteration + BOOL Success = + GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer.data(), (DWORD)(FileInfoBuffer.size() * sizeof(uint64_t))); + FibClass = FileIdBothDirectoryInfo; // Set up for next iteration uint64_t EntryOffset = 0; @@ -1213,7 +1455,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr for (;;) { const FILE_ID_BOTH_DIR_INFO* DirInfo = - reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset); + reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer.data()) + EntryOffset); std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t)); @@ -1338,6 +1580,172 @@ CanonicalPath(std::filesystem::path InPath, std::error_code& Ec) #endif } +bool +IsFile(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = IsFile(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a file", Path.string())); + } + return Result; +} + +bool +IsFile(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == 0; +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + if (S_ISREG(Stat.st_mode)) + { + return true; + } + return false; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +IsDir(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = IsDir(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a directory", Path.string())); + } + return Result; +} + +bool +IsDir(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY; +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + if (S_ISDIR(Stat.st_mode)) + { + return true; + } + return false; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveFile(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Success = RemoveFile(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to remove file '{}'", Path.string())); + } + return Success; +} + +bool +RemoveFile(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + return RemoveFileNative(Path, false, Ec); +#else + bool IsDirectory = std::filesystem::is_directory(Path, Ec); + if (IsDirectory) + { + Ec = MakeErrorCode(EPERM); + return false; + } + Ec.clear(); + return RemoveFileNative(Path, false, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveDir(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Success = RemoveDir(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to remove directory '{}'", Path.string())); + } + return Success; +} + +bool +RemoveDir(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + return RemoveDirNative(Path, Ec); +#else + bool IsFile = std::filesystem::is_regular_file(Path, Ec); + if (IsFile) + { + Ec = MakeErrorCode(EPERM); + return false; + } + Ec.clear(); + return RemoveDirNative(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec) { @@ -1435,6 +1843,49 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec) } uint64_t +FileSizeFromPath(const std::filesystem::path& Path) +{ + std::error_code Ec; + uint64_t Size = FileSizeFromPath(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to get file size for path '{}'", Path.string())); + } + return Size; +} + +uint64_t +FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + void* Handle = ::CreateFile(Path.native().c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + DWORD LastError = GetLastError(); + Ec = MakeErrorCode(LastError); + return 0; + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); + LARGE_INTEGER FileSize; + BOOL Success = GetFileSizeEx(Handle, &FileSize); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + return 0; + } + return FileSize.QuadPart; +#else + return std::filesystem::file_size(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +uint64_t FileSizeFromHandle(void* NativeHandle) { uint64_t FileSize = ~0ull; @@ -1483,7 +1934,13 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) // PathFromHandle void* Handle; #if ZEN_PLATFORM_WINDOWS - Handle = CreateFileW(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + Handle = CreateFileW(Filename.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); if (Handle == INVALID_HANDLE_VALUE) { ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); @@ -1493,7 +1950,7 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) uint64_t ModificatonTick = GetModificationTickFromHandle(Handle, Ec); if (Ec) { - ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error(Ec, fmt::format("Failed to get modification tick for path '{}'", Filename.string())); } return ModificatonTick; #else @@ -1507,6 +1964,56 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) #endif } +void +RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) +{ + std::error_code Ec; + RenameFile(SourcePath, TargetPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to rename path from '{}' to '{}'", SourcePath.string(), TargetPath.string())); + } +} + +void +RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::MoveFileEx(SourcePath.native().c_str(), TargetPath.native().c_str(), MOVEFILE_REPLACE_EXISTING); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + } +#else + return std::filesystem::rename(SourcePath, TargetPath, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +void +RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) +{ + std::error_code Ec; + RenameDirectory(SourcePath, TargetPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to rename directory from '{}' to '{}'", SourcePath.string(), TargetPath.string())); + } +} + +void +RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::MoveFile(SourcePath.native().c_str(), TargetPath.native().c_str()); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + } +#else + return std::filesystem::rename(SourcePath, TargetPath, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + std::filesystem::path GetRunningExecutablePath() { @@ -1793,7 +2300,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) }; auto IsEmpty = [](const std::filesystem::path& Path, std::error_code& Ec) -> bool { - bool Exists = std::filesystem::exists(Path, Ec); + bool Exists = IsFile(Path, Ec); if (Ec) { return false; @@ -1802,7 +2309,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) { return true; } - uintmax_t Size = std::filesystem::file_size(Path, Ec); + uintmax_t Size = FileSizeFromPath(Path, Ec); if (Ec) { return false; @@ -1821,17 +2328,17 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) for (auto i = MaxFiles; i > 0; i--) { std::filesystem::path src = GetFileName(i - 1); - if (!std::filesystem::exists(src)) + if (!IsFile(src)) { continue; } std::error_code DummyEc; std::filesystem::path target = GetFileName(i); - if (std::filesystem::exists(target, DummyEc)) + if (IsFile(target, DummyEc)) { - std::filesystem::remove(target, DummyEc); + RemoveFile(target, DummyEc); } - std::filesystem::rename(src, target, DummyEc); + RenameFile(src, target, DummyEc); } } @@ -1868,16 +2375,16 @@ RotateDirectories(const std::filesystem::path& DirectoryName, std::size_t MaxDir { const std::filesystem::path SourcePath = GetPathForIndex(i - 1); - if (std::filesystem::exists(SourcePath)) + if (IsDir(SourcePath)) { std::filesystem::path TargetPath = GetPathForIndex(i); std::error_code DummyEc; - if (std::filesystem::exists(TargetPath, DummyEc)) + if (IsDir(TargetPath, DummyEc)) { - std::filesystem::remove_all(TargetPath, DummyEc); + DeleteDirectories(TargetPath, DummyEc); } - std::filesystem::rename(SourcePath, TargetPath, DummyEc); + RenameDirectory(SourcePath, TargetPath, DummyEc); } } @@ -1936,22 +2443,46 @@ PickDefaultSystemRootDirectory() #if ZEN_PLATFORM_WINDOWS uint32_t -GetFileAttributes(const std::filesystem::path& Filename) +GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec) { DWORD Attributes = ::GetFileAttributes(Filename.native().c_str()); if (Attributes == INVALID_FILE_ATTRIBUTES) { - ThrowLastError(fmt::format("failed to get attributes of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + return 0; } return (uint32_t)Attributes; } +uint32_t +GetFileAttributes(const std::filesystem::path& Filename) +{ + std::error_code Ec; + uint32_t Result = zen::GetFileAttributes(Filename, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to get attributes of file '{}'", Filename.string())); + } + return Result; +} + void -SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec) { if (::SetFileAttributes(Filename.native().c_str(), Attributes) == 0) { - ThrowLastError(fmt::format("failed to set attributes of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + } +} + +void +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +{ + std::error_code Ec; + zen::SetFileAttributes(Filename, Attributes, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to set attributes of file {}", Filename.string())); } } @@ -1962,87 +2493,103 @@ SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) uint32_t GetFileMode(const std::filesystem::path& Filename) { + std::error_code Ec; + uint32_t Result = GetFileMode(Filename, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to get mode of file {}", Filename)); + } + return Result; +} + +uint32_t +GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec) +{ struct stat Stat; int err = stat(Filename.native().c_str(), &Stat); if (err) { - ThrowLastError(fmt::format("Failed to get mode of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + return 0; } return (uint32_t)Stat.st_mode; } void -SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes) +SetFileMode(const std::filesystem::path& Filename, uint32_t Mode) { - int err = chmod(Filename.native().c_str(), (mode_t)Attributes); - if (err) + std::error_code Ec; + SetFileMode(Filename, Mode, Ec); + if (Ec) { - ThrowLastError(fmt::format("Failed to set mode of file {}", Filename)); + throw std::system_error(Ec, fmt::format("Failed to set mode of file {}", Filename)); } } -#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - -#if ZEN_PLATFORM_WINDOWS -const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; -#else -const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; -#endif // ZEN_PLATFORM_WINDOWS - -const uint32_t FileModeWriteEnableFlags = 0222; - -bool -IsFileAttributeReadOnly(uint32_t FileAttributes) -{ -#if ZEN_PLATFORM_WINDOWS - return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; -#else - return (FileAttributes & 0x00000001) != 0; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -IsFileModeReadOnly(uint32_t FileMode) -{ - return (FileMode & FileModeWriteEnableFlags) == 0; -} - -uint32_t -MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +void +SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec) { - return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); + int err = chmod(Filename.native().c_str(), (mode_t)Mode); + if (err) + { + Ec = MakeErrorCodeFromLastError(); + } } -uint32_t -MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) -{ - return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); -} +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool -SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS - uint32_t CurrentAttributes = GetFileAttributes(Filename); - uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); + uint32_t CurrentAttributes = GetFileAttributes(Filename, Ec); + if (Ec) + { + return false; + } + uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); if (CurrentAttributes != NewAttributes) { - SetFileAttributes(Filename, NewAttributes); + SetFileAttributes(Filename, NewAttributes, Ec); + if (Ec) + { + return false; + } return true; } #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - uint32_t CurrentMode = GetFileMode(Filename); - uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); + uint32_t CurrentMode = GetFileMode(Filename, Ec); + if (Ec) + { + return false; + } + uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); if (CurrentMode != NewMode) { - SetFileMode(Filename, NewMode); + SetFileMode(Filename, NewMode, Ec); + if (Ec) + { + return false; + } return true; } #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC return false; } +bool +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +{ + std::error_code Ec; + bool Result = SetFileReadOnly(Filename, ReadOnly, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to set read only mode of file '{}'", Filename.string())); + } + return Result; +} + std::filesystem::path StringToPath(const std::string_view& Path) { @@ -2079,7 +2626,7 @@ TEST_CASE("filesystem") path BinPath = GetRunningExecutablePath(); const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver"; CHECK(ExpectedExe); - CHECK(is_regular_file(BinPath)); + CHECK(IsFile(BinPath)); // PathFromHandle void* Handle; @@ -2132,6 +2679,80 @@ TEST_CASE("filesystem") CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize()); } +TEST_CASE("Filesystem.Basics") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test"; + CleanDirectory(TestBaseDir, true); + DeleteDirectories(TestBaseDir); + CHECK(!IsDir(TestBaseDir)); + CHECK(CleanDirectory(TestBaseDir, false)); + CHECK(IsDir(TestBaseDir)); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK(!IsDir(TestBaseDir / "no_such_thing")); + CHECK(!IsDir("hgjda/cev_/q12")); + CHECK(!IsFile(TestBaseDir)); + CHECK(!IsFile(TestBaseDir / "no_such_thing")); + CHECK(!IsFile("hgjda/cev_/q12")); + CHECK_THROWS(FileSizeFromPath(TestBaseDir) == 0); + CHECK_THROWS(FileSizeFromPath(TestBaseDir / "no_such_file")); + CHECK(!CreateDirectories(TestBaseDir)); + CHECK(CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(!CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit")); + CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep" / "no")); + CHECK_THROWS(WriteFile(TestBaseDir / "nested" / "a", IoBuffer(20))); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "a" / "yo", IoBuffer(20))); + CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "nested" / "a" / "yo") == 20); + CHECK(!IsFile(TestBaseDir / "nested" / "a")); + CHECK(DeleteDirectories(TestBaseDir / "nested" / "a" / "bit")); + CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit")); + CHECK(!DeleteDirectories(TestBaseDir / "nested" / "a" / "bit")); + CHECK(IsDir(TestBaseDir / "nested" / "a")); + CHECK(DeleteDirectories(TestBaseDir / "nested")); + CHECK(!IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(CreateDirectories(TestBaseDir / "nested" / "deeper")); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "deeper" / "yo", IoBuffer(20))); + CHECK_NOTHROW(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place")); + CHECK(IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20); + CHECK(IsDir(TestBaseDir / "new_place")); + CHECK(!IsFile(TestBaseDir / "new_place")); + CHECK_THROWS(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place")); + CHECK(!RemoveDir(TestBaseDir / "nested" / "deeper")); + CHECK(RemoveFile(TestBaseDir / "new_place" / "yo")); + CHECK(!IsFile(TestBaseDir / "new_place" / "yo")); + CHECK_THROWS(FileSizeFromPath(TestBaseDir / "new_place" / "yo")); + CHECK(!RemoveFile(TestBaseDir / "new_place" / "yo")); + CHECK_THROWS(RemoveFile(TestBaseDir / "nested")); + CHECK_THROWS(RemoveDir(TestBaseDir)); + CHECK_NOTHROW(WriteFile(TestBaseDir / "yo", IoBuffer(20))); + CHECK_NOTHROW(RenameFile(TestBaseDir / "yo", TestBaseDir / "new_place" / "yo")); + CHECK(!IsFile(TestBaseDir / "yo")); + CHECK(IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20); + CHECK_THROWS(RemoveDir(TestBaseDir / "new_place" / "yo")); + CHECK(DeleteDirectories(TestBaseDir)); + CHECK(!IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(!IsDir(TestBaseDir)); + CHECK(!IsDir(TestBaseDir / "nested")); + CHECK(CreateDirectories(TestBaseDir / "nested")); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20))); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true)); + CHECK_THROWS(RemoveFile(TestBaseDir / "nested" / "readonly")); + CHECK_THROWS(CleanDirectory(TestBaseDir, false)); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", false)); + CHECK(RemoveFile(TestBaseDir / "nested" / "readonly")); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20))); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true)); + CHECK(!CleanDirectory(TestBaseDir / "nested", true)); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK(RemoveDir(TestBaseDir)); +} + TEST_CASE("WriteFile") { std::filesystem::path TempFile = GetRunningExecutablePath().parent_path(); @@ -2166,7 +2787,7 @@ TEST_CASE("WriteFile") CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0); } - std::filesystem::remove(TempFile); + RemoveFile(TempFile); } TEST_CASE("DiskSpaceInfo") @@ -2223,7 +2844,7 @@ TEST_CASE("PathBuilder") TEST_CASE("RotateDirectories") { std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test"; - CleanDirectory(TestBaseDir); + CleanDirectory(TestBaseDir, false); std::filesystem::path RotateDir = TestBaseDir / "rotate_dir" / "dir_to_rotate"; IoBuffer DummyFileData = IoBufferBuilder::MakeCloneFromMemory("blubb", 5); @@ -2237,16 +2858,16 @@ TEST_CASE("RotateDirectories") const int RotateMax = 10; NewDir(); - CHECK(std::filesystem::exists(RotateDir)); + CHECK(IsDir(RotateDir)); RotateDirectories(RotateDir, RotateMax); - CHECK(!std::filesystem::exists(RotateDir)); - CHECK(std::filesystem::exists(DirWithSuffix(1))); + CHECK(!IsDir(RotateDir)); + CHECK(IsDir(DirWithSuffix(1))); NewDir(); - CHECK(std::filesystem::exists(RotateDir)); + CHECK(IsDir(RotateDir)); RotateDirectories(RotateDir, RotateMax); - CHECK(!std::filesystem::exists(RotateDir)); - CHECK(std::filesystem::exists(DirWithSuffix(1))); - CHECK(std::filesystem::exists(DirWithSuffix(2))); + CHECK(!IsDir(RotateDir)); + CHECK(IsDir(DirWithSuffix(1))); + CHECK(IsDir(DirWithSuffix(2))); for (int i = 0; i < RotateMax; ++i) { @@ -2256,16 +2877,16 @@ TEST_CASE("RotateDirectories") CHECK_EQ(IsError, false); } - CHECK(!std::filesystem::exists(RotateDir)); + CHECK(!IsDir(RotateDir)); for (int i = 0; i < RotateMax; ++i) { - CHECK(std::filesystem::exists(DirWithSuffix(i + 1))); + CHECK(IsDir(DirWithSuffix(i + 1))); } for (int i = RotateMax; i < RotateMax + 5; ++i) { - CHECK(!std::filesystem::exists(DirWithSuffix(RotateMax + i + 1))); + CHECK(!IsDir(DirWithSuffix(RotateMax + i + 1))); } } diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 9a2b15d1d..c23f16d03 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -20,21 +20,35 @@ class WorkerThreadPool; /** Delete directory (after deleting any contents) */ -ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path); + +/** Delete directory (after deleting any contents) + */ +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists. + + Will also create any required parent direCleanDirectoryctories + */ +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path); /** Ensure directory exists. Will also create any required parent directories */ -ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& Path); /** Map native file handle to a path */ @@ -44,6 +58,46 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_ */ ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); + /** Query file size from native file handle */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); @@ -56,6 +110,22 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro */ ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + ZENCORE_API std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS @@ -277,12 +347,16 @@ std::filesystem::path PickDefaultSystemRootDirectory(); #if ZEN_PLATFORM_WINDOWS uint32_t GetFileAttributes(const std::filesystem::path& Filename); +uint32_t GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec); void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes); +void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC uint32_t GetFileMode(const std::filesystem::path& Filename); -void SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes); +uint32_t GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec); #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool IsFileAttributeReadOnly(uint32_t FileAttributes); @@ -290,6 +364,7 @@ bool IsFileModeReadOnly(uint32_t FileMode); uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly); uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); +bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); std::filesystem::path StringToPath(const std::string_view& Path); diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 0761521dc..2fe5b8948 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -60,7 +60,7 @@ GetPidStatus(int Pid, std::error_code& OutEc) { std::filesystem::path EntryPath = std::filesystem::path("/proc") / fmt::format("{}", Pid); std::filesystem::path StatPath = EntryPath / "stat"; - if (std::filesystem::is_regular_file(StatPath)) + if (IsFile(StatPath)) { FILE* StatFile = fopen(StatPath.c_str(), "r"); if (StatFile) diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp index 641d5508a..9f50de032 100644 --- a/src/zencore/testutils.cpp +++ b/src/zencore/testutils.cpp @@ -4,6 +4,7 @@ #if ZEN_WITH_TESTS +# include <zencore/filesystem.h> # include <zencore/session.h> # include "zencore/string.h" @@ -19,8 +20,8 @@ CreateTemporaryDirectory() std::error_code Ec; std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str(); - std::filesystem::remove_all(DirPath, Ec); - std::filesystem::create_directories(DirPath); + DeleteDirectories(DirPath, Ec); + CreateDirectories(DirPath); return DirPath; } @@ -32,14 +33,14 @@ ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporar ScopedTemporaryDirectory::ScopedTemporaryDirectory(std::filesystem::path Directory) : m_RootPath(Directory) { std::error_code Ec; - std::filesystem::remove_all(Directory, Ec); - std::filesystem::create_directories(Directory); + DeleteDirectories(Directory, Ec); + CreateDirectories(Directory); } ScopedTemporaryDirectory::~ScopedTemporaryDirectory() { std::error_code Ec; - std::filesystem::remove_all(m_RootPath, Ec); + DeleteDirectories(m_RootPath, Ec); } IoBuffer diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp index 1a9892d5c..8f7befc80 100644 --- a/src/zenhttp/auth/authmgr.cpp +++ b/src/zenhttp/auth/authmgr.cpp @@ -379,7 +379,7 @@ private: AuthState.EndArray(); } - std::filesystem::create_directories(m_Config.RootDirectory); + CreateDirectories(m_Config.RootDirectory); std::optional<std::string> Reason; diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 027a35998..78a735ea0 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -3306,18 +3306,18 @@ GenerateFolderContent(const std::filesystem::path& RootPath) std::filesystem::path EmptyFolder(RootPath / "empty_folder"); std::filesystem::path FirstFolder(RootPath / "first_folder"); - std::filesystem::create_directory(FirstFolder); + CreateDirectories(FirstFolder); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22))); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122))); std::filesystem::path SecondFolder(RootPath / "second_folder"); - std::filesystem::create_directory(SecondFolder); + CreateDirectories(SecondFolder); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225))); std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second"); - std::filesystem::create_directory(SecondFolderChild); + CreateDirectories(SecondFolderChild); Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622))); for (const auto& It : Result) @@ -3473,7 +3473,7 @@ TEST_CASE("workspaces.create") while (true) { std::error_code Ec; - std::filesystem::remove_all(Root2Path / Share2Path, Ec); + DeleteDirectories(Root2Path / Share2Path, Ec); if (!Ec) break; } @@ -3630,7 +3630,7 @@ TEST_CASE("workspaces.lifetimes") } // Wipe system config - std::filesystem::remove_all(SystemRootPath); + DeleteDirectories(SystemRootPath); // Restart @@ -3696,8 +3696,8 @@ TEST_CASE("workspaces.share") uint64_t Size = FileObject["size"sv].AsUInt64(); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); - CHECK(std::filesystem::file_size(AbsFilePath) == Size); + CHECK(IsFile(AbsFilePath)); + CHECK(FileSizeFromPath(AbsFilePath) == Size); Files.insert_or_assign(ChunkId, std::make_pair(AbsFilePath, Size)); } } @@ -3720,7 +3720,7 @@ TEST_CASE("workspaces.share") CHECK(ChunkId != Oid::Zero); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); + CHECK(IsFile(AbsFilePath)); } } } @@ -3740,7 +3740,7 @@ TEST_CASE("workspaces.share") CHECK(ChunkId != Oid::Zero); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); + CHECK(IsFile(AbsFilePath)); } } diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 0da6e31ad..73166e608 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -40,7 +40,7 @@ struct DirStats DirStats GetStatsForDirectory(std::filesystem::path Dir) { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) return {}; struct StatsTraversal : public GetDirectoryContentVisitor diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index d3af0c6a6..52f539dcd 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -1116,7 +1116,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) if (DataDir.empty()) throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot"); - if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir)) + if (!IsDir(ServerOptions.BaseSnapshotDir)) throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir)); } diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 31d9e1c94..104b26954 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -50,7 +50,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { break; } - if (std::filesystem::is_regular_file(ParentPath / "xmake.lua", ErrorCode)) + if (IsFile(ParentPath / "xmake.lua", ErrorCode)) { if (ErrorCode) { @@ -59,7 +59,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di std::filesystem::path HtmlDir = ParentPath / "src" / "zenserver" / "frontend" / "html"; - if (std::filesystem::is_directory(HtmlDir, ErrorCode)) + if (IsDir(HtmlDir, ErrorCode)) { m_Directory = HtmlDir; } diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index d5419d342..78ddd39a0 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -406,17 +406,17 @@ main(int argc, char* argv[]) if (!DeleteReason.empty()) { - if (std::filesystem::exists(ServerOptions.DataDir)) + if (IsDir(ServerOptions.DataDir)) { ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); DeleteDirectories(ServerOptions.DataDir); } } - if (!std::filesystem::exists(ServerOptions.DataDir)) + if (!IsDir(ServerOptions.DataDir)) { ServerOptions.IsFirstRun = true; - std::filesystem::create_directories(ServerOptions.DataDir); + CreateDirectories(ServerOptions.DataDir); } if (!ServerOptions.BaseSnapshotDir.empty()) diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index e757ef84e..5af803617 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -251,7 +251,7 @@ HttpObjectStoreService::Inititalize() ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; - if (!fs::exists(BucketsPath)) + if (!IsDir(BucketsPath)) { CreateDirectories(BucketsPath); } @@ -324,7 +324,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; { std::lock_guard _(BucketsMutex); - if (!fs::exists(BucketPath)) + if (!IsDir(BucketPath)) { CreateDirectories(BucketPath); ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); @@ -406,7 +406,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); FileSystemTraversal Traversal; - if (std::filesystem::exists(FullPath)) + if (IsDir(FullPath)) { std::lock_guard _(BucketsMutex); Traversal.TraverseFileSystem(FullPath, FileVisitor); @@ -475,7 +475,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st } const fs::path FilePath = BucketDir / RelativeBucketPath; - if (!fs::exists(FilePath)) + if (!IsFile(FilePath)) { ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); @@ -576,7 +576,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) { std::lock_guard _(BucketsMutex); - if (!fs::exists(FileDirectory)) + if (!IsDir(FileDirectory)) { CreateDirectories(FileDirectory); } diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp index 98e292d91..375e44e59 100644 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp @@ -73,7 +73,7 @@ public: ContainerObject.IterateAttachments([&](CbFieldView FieldView) { IoHash AttachmentHash = FieldView.AsBinaryAttachment(); std::filesystem::path AttachmentPath = GetAttachmentPath(AttachmentHash); - if (!std::filesystem::exists(AttachmentPath)) + if (!IsFile(AttachmentPath)) { Result.Needs.insert(AttachmentHash); } @@ -111,7 +111,7 @@ public: Stopwatch Timer; SaveAttachmentResult Result; std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::exists(ChunkPath)) + if (!IsFile(ChunkPath)) { try { @@ -182,7 +182,7 @@ public: for (const IoHash& RawHash : BlockHashes) { std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (std::filesystem::is_regular_file(ChunkPath)) + if (IsFile(ChunkPath)) { ExistingBlockHashes.push_back(RawHash); } @@ -203,7 +203,7 @@ public: Stopwatch Timer; LoadAttachmentResult Result; std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::is_regular_file(ChunkPath)) + if (!IsFile(ChunkPath)) { Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); @@ -246,7 +246,7 @@ private: LoadContainerResult Result; std::filesystem::path SourcePath = m_OutputPath; SourcePath.append(Name); - if (!std::filesystem::is_regular_file(SourcePath)) + if (!IsFile(SourcePath)) { Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string()); diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index ea3f2aad9..1966eeef9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -58,7 +58,7 @@ namespace { std::filesystem::path DroppedBucketPath; do { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) { return true; } @@ -68,7 +68,7 @@ namespace { std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), MovedId); DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { if (!DeleteDirectories(DroppedBucketPath)) { @@ -77,7 +77,7 @@ namespace { Dir); continue; } - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { ZEN_INFO("Drop directory '{}' for '{}' still exists after remove, attempting different name.", DroppedBucketPath, Dir); continue; @@ -88,13 +88,13 @@ namespace { do { std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); + RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { OutDeleteDir = DroppedBucketPath; return true; } - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { ZEN_INFO("Can't rename '{}' to still existing drop directory '{}'. Reason: '{}'. Attempting different name.", Dir, @@ -486,7 +486,7 @@ struct ProjectStore::OplogStorage : public RefCounted [[nodiscard]] bool Exists() const { return Exists(m_OplogStoragePath); } [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath) { - return std::filesystem::exists(GetLogPath(BasePath)) && std::filesystem::exists(GetBlobsPath(BasePath)); + return IsFile(GetLogPath(BasePath)) && IsFile(GetBlobsPath(BasePath)); } [[nodiscard]] bool IsValid() const { return IsValid(m_OplogStoragePath); } [[nodiscard]] static bool IsValid(const std::filesystem::path& BasePath) @@ -496,13 +496,13 @@ struct ProjectStore::OplogStorage : public RefCounted void WipeState() const { std::error_code Ec; - std::filesystem::remove(GetLogPath(), Ec); - std::filesystem::remove(GetBlobsPath(), Ec); + RemoveFile(GetLogPath(), Ec); + RemoveFile(GetBlobsPath(), Ec); } static bool Delete(const std::filesystem::path& BasePath) { return DeleteDirectories(BasePath); } - uint64_t OpBlobsSize() const { return std::filesystem::file_size(GetBlobsPath()); } + uint64_t OpBlobsSize() const { return FileSizeFromPath(GetBlobsPath()); } uint64_t OpsSize() const { return OpsSize(m_OplogStoragePath); } static uint64_t OpsSize(const std::filesystem::path& BasePath) @@ -510,7 +510,7 @@ struct ProjectStore::OplogStorage : public RefCounted if (Exists(BasePath)) { std::error_code DummyEc; - return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath)); + return FileSizeFromPath(GetLogPath(BasePath)) + FileSizeFromPath(GetBlobsPath(BasePath)); } return 0; } @@ -689,7 +689,7 @@ struct ProjectStore::OplogStorage : public RefCounted m_OpBlobs.Close(); Oplog.Close(); - std::filesystem::rename(OplogPath, GetLogPath(), Ec); + RenameFile(OplogPath, GetLogPath(), Ec); if (Ec) { throw std::system_error( @@ -702,9 +702,9 @@ struct ProjectStore::OplogStorage : public RefCounted if (Ec) { // We failed late - clean everything up as best we can - std::filesystem::remove(OpBlobs.GetPath(), Ec); - std::filesystem::remove(GetLogPath(), Ec); - std::filesystem::remove(GetBlobsPath(), Ec); + RemoveFile(OpBlobs.GetPath(), Ec); + RemoveFile(GetLogPath(), Ec); + RemoveFile(GetBlobsPath(), Ec); throw std::system_error(Ec, fmt::format("Oplog::Compact failed to rename temporary oplog file from '{}' to '{}'", OpBlobs.GetPath(), @@ -739,7 +739,7 @@ struct ProjectStore::OplogStorage : public RefCounted } catch (const std::exception& /*Ex*/) { - std::filesystem::remove(OpBlobs.GetPath(), Ec); + RemoveFile(OpBlobs.GetPath(), Ec); throw; } } @@ -1108,7 +1108,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id, ZEN_WARN("Invalid oplog found at '{}'. Wiping state for oplog.", m_BasePath); m_Storage->WipeState(); std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } } m_Storage->Open(/* IsCreate */ !StoreExists); @@ -1116,7 +1116,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id, m_MetaPath = m_BasePath / "ops.meta"sv; m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath()); - CleanDirectory(m_TempPath); + CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false); } ProjectStore::Oplog::~Oplog() @@ -1142,7 +1142,7 @@ ProjectStore::Oplog::Flush() if (!m_MetaValid) { std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } uint64_t LogCount = m_Storage->LogCount(); @@ -1238,19 +1238,19 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath) uint64_t Size = OplogStorage::OpsSize(BasePath); std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::exists(StateFilePath)) + if (IsFile(StateFilePath)) { - Size += std::filesystem::file_size(StateFilePath); + Size += FileSizeFromPath(StateFilePath); } std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv; - if (std::filesystem::exists(MetaFilePath)) + if (IsFile(MetaFilePath)) { - Size += std::filesystem::file_size(MetaFilePath); + Size += FileSizeFromPath(MetaFilePath); } std::filesystem::path IndexFilePath = BasePath / "ops.zidx"sv; - if (std::filesystem::exists(IndexFilePath)) + if (IsFile(IndexFilePath)) { - Size += std::filesystem::file_size(IndexFilePath); + Size += FileSizeFromPath(IndexFilePath); } return Size; @@ -1303,7 +1303,7 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath) using namespace std::literals; std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - return std::filesystem::is_regular_file(StateFilePath); + return IsFile(StateFilePath); } bool @@ -1337,7 +1337,7 @@ ProjectStore::Oplog::Read() if (!m_MetaValid) { std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } ReadIndexSnapshot(); @@ -1438,7 +1438,7 @@ ProjectStore::Oplog::Reset() m_Storage = new OplogStorage(this, m_BasePath); m_Storage->Open(true); m_MetaValid = false; - CleanDirectory(m_TempPath); + CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false); Write(); } // Erase content on disk @@ -1457,7 +1457,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f using namespace std::literals; std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::is_regular_file(StateFilePath)) + if (IsFile(StateFilePath)) { // ZEN_INFO("oplog '{}/{}': config read from '{}'", m_OuterProject->Identifier, m_OplogId, StateFilePath); @@ -1536,7 +1536,7 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo if (File.Hash == IoHash::Zero) { std::filesystem::path FilePath = m_OuterProject->RootDir / File.ServerPath; - if (!std::filesystem::is_regular_file(FilePath)) + if (!IsFile(FilePath)) { ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); }); HasMissingEntries = true; @@ -1625,10 +1625,10 @@ ProjectStore::Oplog::WriteIndexSnapshot() fs::path TempIndexPath = m_BasePath / "ops.zidx.tmp"; // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to clean up temp snapshot at {}, reason: '{}'", GetOuterProject()->Identifier, @@ -1641,9 +1641,9 @@ ProjectStore::Oplog::WriteIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, TempIndexPath); + RenameFile(IndexPath, TempIndexPath); } // Write the current state of the location map to a new index state @@ -1778,11 +1778,11 @@ ProjectStore::Oplog::WriteIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(TempIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(TempIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to restore old snapshot from {}, reason: '{}'", @@ -1793,10 +1793,10 @@ ProjectStore::Oplog::WriteIndexSnapshot() } } } - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to remove temporary file {}, reason: '{}'", m_OuterProject->Identifier, @@ -1814,7 +1814,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() ZEN_TRACE_CPU("Oplog::ReadIndexSnapshot"); std::filesystem::path IndexPath = m_BasePath / "ops.zidx"; - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint64_t EntryCount = 0; Stopwatch Timer; @@ -3133,7 +3133,7 @@ ProjectStore::Project::~Project() bool ProjectStore::Project::Exists(const std::filesystem::path& BasePath) { - return std::filesystem::exists(BasePath / "Project.zcb"); + return IsFile(BasePath / "Project.zcb"); } void @@ -3207,7 +3207,7 @@ ProjectStore::Project::ReadAccessTimes() using namespace std::literals; std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv; - if (!std::filesystem::exists(ProjectAccessTimesFilePath)) + if (!IsFile(ProjectAccessTimesFilePath)) { return; } @@ -3598,14 +3598,14 @@ ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath) uint64_t Size = 0; std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv; - if (std::filesystem::exists(AccessTimesFilePath)) + if (IsFile(AccessTimesFilePath)) { - Size += std::filesystem::file_size(AccessTimesFilePath); + Size += FileSizeFromPath(AccessTimesFilePath); } std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectFilePath)) + if (IsFile(ProjectFilePath)) { - Size += std::filesystem::file_size(ProjectFilePath); + Size += FileSizeFromPath(ProjectFilePath); } return Size; @@ -3717,7 +3717,7 @@ ProjectStore::Project::IsExpired(const std::string& EntryName, if (!MarkerPath.empty()) { std::error_code Ec; - if (std::filesystem::exists(MarkerPath, Ec)) + if (IsFile(MarkerPath, Ec)) { if (Ec) { @@ -3870,7 +3870,7 @@ void ProjectStore::DiscoverProjects() { ZEN_MEMSCOPE(GetProjectstoreTag()); - if (!std::filesystem::exists(m_ProjectBasePath)) + if (!IsDir(m_ProjectBasePath)) { return; } @@ -3979,7 +3979,7 @@ ProjectStore::StorageSize() const GcStorageSize Result; { - if (std::filesystem::exists(m_ProjectBasePath)) + if (IsDir(m_ProjectBasePath)) { DirectoryContent ProjectsFolderContent; GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent); @@ -3987,7 +3987,7 @@ ProjectStore::StorageSize() const for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories) { std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectStateFilePath)) + if (IsFile(ProjectStateFilePath)) { Result.DiskSize += Project::TotalSize(ProjectBasePath); DirectoryContent DirContent; @@ -7243,7 +7243,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project1FilePath); + RemoveFile(Project1FilePath); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -7272,7 +7272,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project2Oplog1Path); + RemoveFile(Project2Oplog1Path); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -7301,7 +7301,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project2FilePath); + RemoveFile(Project2FilePath); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24), .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24), @@ -8025,7 +8025,7 @@ TEST_CASE("project.store.rpc.getchunks") CompositeBuffer Buffer = Attachment->AsCompositeBinary(); CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)), IoHash::HashBuffer(Buffer)); - CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second)); + CHECK_EQ(Chunk["Size"sv].AsUInt64(), FileSizeFromPath(FilesOpIdAttachments[0].second)); CHECK(!Chunk.FindView("RawSize")); } { diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index a7263da83..f96b3e185 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -1212,7 +1212,7 @@ BuildContainer(CidStore& ChunkStore, { std::string_view ServerPath = View["serverpath"sv].AsString(); std::filesystem::path FilePath = Project.RootDir / ServerPath; - if (!std::filesystem::is_regular_file(FilePath)) + if (!IsFile(FilePath)) { remotestore_impl::ReportMessage( OptionalContext, @@ -3083,9 +3083,9 @@ LoadOplog(CidStore& ChunkStore, OptionalContext]() { auto _ = MakeGuard([&DechunkLatch, &TempFileName] { std::error_code Ec; - if (std::filesystem::exists(TempFileName, Ec)) + if (IsFile(TempFileName, Ec)) { - std::filesystem::remove(TempFileName, Ec); + RemoveFile(TempFileName, Ec); if (Ec) { ZEN_INFO("Failed to remove temporary file '{}'. Reason: {}", TempFileName, Ec.message()); diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 0b7fd0400..ac0aaef8e 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -1100,7 +1100,7 @@ HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& Workspace } } - if (!std::filesystem::is_directory(Workspace.RootPath / NewConfig.SharePath)) + if (!IsDir(Workspace.RootPath / NewConfig.SharePath)) { return ServerRequest.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 03e269d49..45c91d691 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -434,7 +434,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) { std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; - if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath)) + if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) { ZEN_INFO( "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " @@ -483,7 +483,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) { ZEN_INFO("Deleting '{}'", DirEntry.path()); - std::filesystem::remove_all(DirEntry.path(), Ec); + DeleteDirectories(DirEntry.path(), Ec); if (Ec) { @@ -914,7 +914,7 @@ ZenServer::CheckStateMarker() std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; try { - if (!std::filesystem::exists(StateMarkerPath)) + if (!IsFile(StateMarkerPath)) { ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); RequestExit(1); diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 63c0388fa..7cc09be15 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -85,7 +85,7 @@ BlockStoreFile::Create(uint64_t InitialSize) ZEN_TRACE_CPU("BlockStoreFile::Create"); auto ParentPath = m_Path.parent_path(); - if (!std::filesystem::is_directory(ParentPath)) + if (!IsDir(ParentPath)) { CreateDirectories(ParentPath); } @@ -215,7 +215,7 @@ IsMetaDataValid(const std::filesystem::path& BlockPath, const std::filesystem::p } if (MetaWriteTime < BlockWriteTime) { - std::filesystem::remove(MetaPath, Ec); + RemoveFile(MetaPath, Ec); return false; } return true; @@ -239,7 +239,7 @@ BlockStoreFile::MetaSize() const if (IsMetaDataValid(m_Path, MetaPath)) { std::error_code DummyEc; - if (uint64_t Size = std::filesystem::file_size(MetaPath, DummyEc); !DummyEc) + if (uint64_t Size = FileSizeFromPath(MetaPath, DummyEc); !DummyEc) { return Size; } @@ -252,7 +252,7 @@ BlockStoreFile::RemoveMeta() { std::filesystem::path MetaPath = GetMetaPath(); std::error_code DummyEc; - std::filesystem::remove(MetaPath, DummyEc); + RemoveFile(MetaPath, DummyEc); } std::filesystem::path @@ -291,7 +291,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max m_MaxBlockSize = MaxBlockSize; m_MaxBlockCount = MaxBlockCount; - if (std::filesystem::is_directory(m_BlocksBasePath)) + if (IsDir(m_BlocksBasePath)) { uint32_t NextBlockIndex = 0; std::vector<std::filesystem::path> FoldersToScan; @@ -500,7 +500,7 @@ BlockStore::GetFreeBlockIndex(uint32_t ProbeIndex, RwLock::ExclusiveLockScope&, { OutBlockPath = GetBlockPath(m_BlocksBasePath, ProbeIndex); std::error_code Ec; - bool Exists = std::filesystem::exists(OutBlockPath, Ec); + bool Exists = IsFile(OutBlockPath, Ec); if (Ec) { ZEN_WARN("Failed to probe existence of file '{}' when trying to allocate a new block. Reason: '{}'", @@ -1375,14 +1375,14 @@ TEST_CASE("blockstore.blockfile") BoopChunk = File1.GetChunk(5, 5); } - CHECK(std::filesystem::exists(RootDirectory / "1")); + CHECK(IsFile(RootDirectory / "1")); const char* Data = static_cast<const char*>(DataChunk.GetData()); CHECK(std::string(Data) == "data"); const char* Boop = static_cast<const char*>(BoopChunk.GetData()); CHECK(std::string(Boop) == "boop"); } - CHECK(std::filesystem::exists(RootDirectory / "1")); + CHECK(IsFile(RootDirectory / "1")); { IoBuffer DataChunk; @@ -1401,7 +1401,7 @@ TEST_CASE("blockstore.blockfile") const char* Boop = static_cast<const char*>(BoopChunk.GetData()); CHECK(std::string(Boop) == "boop"); } - CHECK(!std::filesystem::exists(RootDirectory / "1")); + CHECK(!IsFile(RootDirectory / "1")); } namespace blockstore::impl { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index eb36be049..f26901458 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -71,7 +71,7 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) { std::filesystem::path BlobLogPath = blobstore::impl::GetBlobLogPath(Config.RootDirectory); std::filesystem::path MetaLogPath = blobstore::impl::GetMetaLogPath(Config.RootDirectory); - bool IsNew = !(std::filesystem::exists(BlobLogPath) && std::filesystem::exists(MetaLogPath)); + bool IsNew = !(IsFile(BlobLogPath) && IsFile(MetaLogPath)); if (!IsNew) { @@ -501,7 +501,7 @@ uint64_t BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) { ZEN_TRACE_CPU("BuildStore::ReadPayloadLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -518,7 +518,7 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys TCasLogFile<PayloadDiskEntry> CasLog; if (!CasLog.IsValid(LogPath)) { - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } CasLog.Open(LogPath, CasLogFile::Mode::kRead); @@ -603,7 +603,7 @@ uint64_t BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) { ZEN_TRACE_CPU("BuildStore::ReadMetadataLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -620,7 +620,7 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy TCasLogFile<MetadataDiskEntry> CasLog; if (!CasLog.IsValid(LogPath)) { - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } CasLog.Open(LogPath, CasLogFile::Mode::kRead); diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 61552fafc..b2d2416be 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -200,21 +200,21 @@ namespace cache::impl { int DropIndex = 0; do { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) { return false; } std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex); std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { DropIndex++; continue; } std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); + RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { DeleteDirectories(DroppedBucketPath); @@ -909,16 +909,16 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, { std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName); - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { - if (!std::filesystem::remove(LogPath, Ec) || Ec) + if (!RemoveFile(LogPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); std::error_code RemoveIndexEc; - std::filesystem::remove(IndexPath, RemoveIndexEc); + RemoveFile(IndexPath, RemoveIndexEc); } } } @@ -939,7 +939,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const { ZEN_TRACE_CPU("Z$::Bucket::ReadIndexFile"); - if (!std::filesystem::is_regular_file(IndexPath)) + if (!IsFile(IndexPath)) { return 0; } @@ -1023,7 +1023,7 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std:: { ZEN_TRACE_CPU("Z$::Bucket::ReadLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -1103,37 +1103,37 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco if (IsNew) { - fs::remove(LogPath); - fs::remove(IndexPath); - fs::remove_all(m_BlocksBasePath); + RemoveFile(LogPath); + RemoveFile(IndexPath); + DeleteDirectories(m_BlocksBasePath); } CreateDirectories(m_BucketDir); m_BlockStore.Initialize(m_BlocksBasePath, m_Configuration.MaxBlockSize, BlockStoreDiskLocation::MaxBlockIndex + 1); - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexLock, IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile<DiskIndexEntry>::IsValid(LogPath)) { LogEntryCount = ReadLog(IndexLock, LogPath, m_LogFlushPosition); } - else if (fs::is_regular_file(LogPath)) + else if (IsFile(LogPath)) { ZEN_WARN("removing invalid log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } @@ -2146,7 +2146,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); std::error_code Ec; - uintmax_t size = std::filesystem::file_size(DataFilePath.ToPath(), Ec); + uintmax_t size = FileSizeFromPath(DataFilePath.ToPath(), Ec); if (Ec) { ReportBadKey(HashKey); @@ -2287,11 +2287,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) BuildPath(Path, Entry.Key); fs::path FilePath = Path.ToPath(); RwLock::ExclusiveLockScope ValueLock(LockForHash(Entry.Key)); - if (fs::is_regular_file(FilePath)) + if (IsFile(FilePath)) { ZEN_DEBUG("deleting bad standalone cache file '{}'", Path.ToUtf8()); std::error_code Ec; - fs::remove(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... + RemoveFile(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... } } } @@ -2424,7 +2424,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c if (CleanUpTempFile) { std::error_code Ec; - std::filesystem::remove(DataFile.GetPath(), Ec); + RemoveFile(DataFile.GetPath(), Ec); if (Ec) { ZEN_WARN("Failed to clean up temporary file '{}' for put in '{}', reason '{}'", @@ -2452,7 +2452,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); // We do a speculative remove of the file instead of probing with a exists call and check the error code instead - std::filesystem::remove(FsPath, Ec); + RemoveFile(FsPath, Ec); if (Ec) { if (Ec.value() != ENOENT) @@ -2460,7 +2460,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c ZEN_WARN("Failed to remove file '{}' for put in '{}', reason: '{}', retrying.", FsPath, m_BucketDir, Ec.message()); Sleep(100); Ec.clear(); - std::filesystem::remove(FsPath, Ec); + RemoveFile(FsPath, Ec); if (Ec && Ec.value() != ENOENT) { throw std::system_error(Ec, fmt::format("Failed to remove file '{}' for put in '{}'", FsPath, m_BucketDir)); @@ -2791,7 +2791,7 @@ public: ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': deleting standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8()); std::error_code Ec; - if (!fs::remove(FilePath, Ec)) + if (!RemoveFile(FilePath, Ec)) { continue; } @@ -2812,7 +2812,7 @@ public: ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': checking standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8()); std::error_code Ec; - bool Existed = std::filesystem::is_regular_file(FilePath, Ec); + bool Existed = IsFile(FilePath, Ec); if (Ec) { ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': failed checking cache payload file '{}'. Reason '{}'", diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 73c10a6db..ed42f254e 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -118,7 +118,7 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig) // Ensure root directory exists - create if it doesn't exist already - std::filesystem::create_directories(m_Config.RootDirectory); + CreateDirectories(m_Config.RootDirectory); // Open or create manifest diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp index 6c7b1b297..492ce9317 100644 --- a/src/zenstore/caslog.cpp +++ b/src/zenstore/caslog.cpp @@ -37,7 +37,7 @@ CasLogFile::~CasLogFile() bool CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize) { - if (!std::filesystem::is_regular_file(FileName)) + if (!IsFile(FileName)) { return false; } diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index b64bc26dd..184251da7 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -927,10 +927,10 @@ CasContainerStrategy::MakeIndexSnapshot() fs::path TempIndexPath = cas::impl::GetTempIndexPath(m_RootDirectory, m_ContainerBaseName); // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", TempIndexPath, Ec.message()); return; @@ -939,9 +939,9 @@ CasContainerStrategy::MakeIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, TempIndexPath); + RenameFile(IndexPath, TempIndexPath); } // Write the current state of the location map to a new index state @@ -992,21 +992,21 @@ CasContainerStrategy::MakeIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(TempIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(TempIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", TempIndexPath, Ec.message()); } } } - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempIndexPath, Ec.message()); } @@ -1092,7 +1092,7 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski if (!TCasLogFile<CasDiskIndexEntry>::IsValid(LogPath)) { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } @@ -1155,7 +1155,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) if (IsNewStore) { - std::filesystem::remove_all(BasePath); + DeleteDirectories(BasePath); } CreateDirectories(BasePath); @@ -1165,19 +1165,19 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) std::filesystem::path LogPath = cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName); std::filesystem::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName); - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile<CasDiskIndexEntry>::IsValid(LogPath)) { @@ -1186,7 +1186,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) else { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 34db51aa9..14bdc41f0 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -176,10 +176,10 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN if (IsNewStore) { - std::filesystem::remove(LogPath); - std::filesystem::remove(IndexPath); + RemoveFile(LogPath); + RemoveFile(IndexPath); - if (std::filesystem::is_directory(m_RootDirectory)) + if (IsDir(m_RootDirectory)) { // We need to explicitly only delete sharded root folders as the cas manifest, tinyobject and smallobject cas folders may reside // in this folder as well @@ -211,24 +211,24 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN Traversal.TraverseFileSystem(m_RootDirectory, CasVisitor); for (const std::filesystem::path& SharededRoot : CasVisitor.ShardedRoots) { - std::filesystem::remove_all(SharededRoot); + DeleteDirectories(SharededRoot); } } } - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile<FileCasIndexEntry>::IsValid(LogPath)) { @@ -237,7 +237,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN else { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } @@ -327,7 +327,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: { std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); std::error_code Ec; - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("unable to move existing CAS file {} to {}", ChunkPath, TempPath)); @@ -452,7 +452,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: { PayloadFile.Close(); std::error_code DummyEc; - std::filesystem::remove(ChunkPath, DummyEc); + RemoveFile(ChunkPath, DummyEc); throw; } bool IsNew = UpdateIndex(ChunkHash, Chunk.Size()); @@ -503,7 +503,7 @@ FileCasStrategy::SafeOpenChunk(const IoHash& ChunkHash, uint64 ExpectedSize) { std::error_code Ec; std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (!Ec) { Chunk.SetDeleteOnClose(true); @@ -574,7 +574,7 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) ShardingHelper Name(m_RootDirectory, ChunkHash); const std::filesystem::path ChunkPath = Name.ShardedPath.ToPath(); - uint64_t FileSize = static_cast<uint64_t>(std::filesystem::file_size(ChunkPath, Ec)); + uint64_t FileSize = static_cast<uint64_t>(FileSizeFromPath(ChunkPath, Ec)); if (Ec) { ZEN_WARN("get file size FAILED, file cas '{}'", ChunkPath); @@ -582,9 +582,9 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) } ZEN_DEBUG("deleting CAS payload file '{}' {}", ChunkPath, NiceBytes(FileSize)); - std::filesystem::remove(ChunkPath, Ec); + RemoveFile(ChunkPath, Ec); - if (!Ec || !std::filesystem::exists(ChunkPath)) + if (!Ec || !IsFile(ChunkPath)) { { RwLock::ExclusiveLockScope _(m_Lock); @@ -941,10 +941,10 @@ FileCasStrategy::MakeIndexSnapshot() fs::path STmpIndexPath = GetTempIndexPath(m_RootDirectory); // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - if (!fs::remove(STmpIndexPath, Ec) || Ec) + if (!RemoveFile(STmpIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message()); return; @@ -953,9 +953,9 @@ FileCasStrategy::MakeIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, STmpIndexPath); + RenameFile(IndexPath, STmpIndexPath); } // Write the current state of the location map to a new index state @@ -1004,21 +1004,21 @@ FileCasStrategy::MakeIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(STmpIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(STmpIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message()); } } } - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - if (!fs::remove(STmpIndexPath, Ec) || Ec) + if (!RemoveFile(STmpIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message()); } @@ -1032,7 +1032,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& using namespace filecas::impl; std::vector<FileCasIndexEntry> Entries; - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -1077,7 +1077,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& return 0; } - if (std::filesystem::is_directory(m_RootDirectory)) + if (IsDir(m_RootDirectory)) { ZEN_INFO("missing index for file cas, scanning for cas files in {}", m_RootDirectory); TCasLogFile<FileCasIndexEntry> CasLog; @@ -1116,7 +1116,7 @@ FileCasStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntr using namespace filecas::impl; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { uint64_t LogEntryCount = 0; Stopwatch Timer; @@ -1274,12 +1274,12 @@ public: ChunkPath); } std::error_code Ec; - uint64_t SizeOnDisk = std::filesystem::file_size(ChunkPath, Ec); + uint64_t SizeOnDisk = FileSizeFromPath(ChunkPath, Ec); if (Ec) { SizeOnDisk = 0; } - bool Existed = std::filesystem::remove(ChunkPath, Ec); + bool Existed = RemoveFile(ChunkPath, Ec); if (Ec) { // Target file may be open for read, attempt to move it to a temp file and mark it delete on close @@ -1290,7 +1290,7 @@ public: if (OldChunk) { std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (!Ec) { OldChunk.SetDeleteOnClose(true); @@ -1317,7 +1317,7 @@ public: else { std::error_code Ec; - bool Existed = std::filesystem::is_regular_file(ChunkPath, Ec); + bool Existed = IsFile(ChunkPath, Ec); if (Ec) { if (Ctx.Settings.Verbose) @@ -1516,7 +1516,7 @@ TEST_CASE("cas.chunk.moveoverwrite") Payload1.SetDeleteOnClose(true); CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash()); CHECK_EQ(Result.New, true); - CHECK(!std::filesystem::exists(Payload1Path)); + CHECK(!IsFile(Payload1Path)); } { std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"}; @@ -1526,9 +1526,9 @@ TEST_CASE("cas.chunk.moveoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash()); CHECK_EQ(Result.New, false); - CHECK(std::filesystem::exists(Payload1BPath)); + CHECK(IsFile(Payload1BPath)); Payload1B = {}; - CHECK(!std::filesystem::exists(Payload1BPath)); + CHECK(!IsFile(Payload1BPath)); } IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash()); @@ -1554,7 +1554,7 @@ TEST_CASE("cas.chunk.moveoverwrite") } Payload2 = {}; - CHECK(!std::filesystem::exists(Payload2Path)); + CHECK(!IsFile(Payload2Path)); { IoHash RawHash; @@ -1598,9 +1598,9 @@ TEST_CASE("cas.chunk.copyoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly); CHECK_EQ(Result.New, true); - CHECK(std::filesystem::exists(Payload1Path)); + CHECK(IsFile(Payload1Path)); Payload1 = {}; - CHECK(!std::filesystem::exists(Payload1Path)); + CHECK(!IsFile(Payload1Path)); } { std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"}; @@ -1611,9 +1611,9 @@ TEST_CASE("cas.chunk.copyoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly); CHECK_EQ(Result.New, false); - CHECK(std::filesystem::exists(Payload1BPath)); + CHECK(IsFile(Payload1BPath)); Payload1B = {}; - CHECK(!std::filesystem::exists(Payload1BPath)); + CHECK(!IsFile(Payload1BPath)); } IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash()); @@ -1640,7 +1640,7 @@ TEST_CASE("cas.chunk.copyoverwrite") } Payload2 = {}; - CHECK(!std::filesystem::exists(Payload2Path)); + CHECK(!IsFile(Payload2Path)); { IoHash RawHash; diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index fe5ae284b..ac4dda83f 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -62,11 +62,11 @@ namespace { { if (Size == 0) { - std::filesystem::remove(Path); + RemoveFile(Path); return std::error_code{}; } CreateDirectories(Path.parent_path()); - if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size) + if (IsFile(Path) && FileSizeFromPath(Path) == Size) { return std::error_code(); } @@ -1262,12 +1262,12 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_TRACE_CPU("GcV2::CompactStores"); auto ClaimDiskReserve = [&]() -> uint64_t { - if (!std::filesystem::is_regular_file(Settings.DiskReservePath)) + if (!IsFile(Settings.DiskReservePath)) { return 0; } - uint64_t ReclaimedSize = std::filesystem::file_size(Settings.DiskReservePath); - if (std::filesystem::remove(Settings.DiskReservePath)) + uint64_t ReclaimedSize = FileSizeFromPath(Settings.DiskReservePath); + if (RemoveFile(Settings.DiskReservePath)) { return ReclaimedSize; } @@ -1557,7 +1557,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) m_Config.LightweightInterval = m_Config.MonitorInterval; } - std::filesystem::create_directories(Config.RootDirectory); + CreateDirectories(Config.RootDirectory); std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); if (Ec) @@ -1850,7 +1850,7 @@ GcScheduler::GetState() const if (Result.Config.DiskReserveSize != 0) { Ec.clear(); - Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec; + Result.HasDiskReserve = IsFile(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec; } if (Result.Status != GcSchedulerStatus::kRunning) @@ -2389,12 +2389,12 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, { // We are low on disk, check if we can release our extra storage reserve, if we can't bail from doing GC auto ClaimDiskReserve = [&]() -> uint64_t { - if (!std::filesystem::is_regular_file(DiskReservePath)) + if (!IsFile(DiskReservePath)) { return 0; } - uint64_t ReclaimedSize = std::filesystem::file_size(DiskReservePath); - if (std::filesystem::remove(DiskReservePath)) + uint64_t ReclaimedSize = FileSizeFromPath(DiskReservePath); + if (RemoveFile(DiskReservePath)) { return ReclaimedSize; } diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index 02a83d2a6..0ca2adab2 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -444,7 +444,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) { const std::filesystem::path& RootPath = Workspace->GetConfig().RootPath; std::filesystem::path ConfigPath = RootPath / WorkspaceConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::string Error; std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error); @@ -458,7 +458,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) { const std::filesystem::path& SharePath = Configuration.SharePath; - if (std::filesystem::is_directory(RootPath / SharePath)) + if (IsDir(RootPath / SharePath)) { DeletedShares.erase(Configuration.Id); @@ -808,7 +808,7 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work ZEN_DEBUG("Reading workspaces state from {}", WorkspaceStatePath); const std::filesystem::path ConfigPath = WorkspaceStatePath / WorkspacesConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::vector<Workspaces::WorkspaceConfiguration> Workspaces = WorkspacesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); @@ -847,7 +847,7 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p ZEN_DEBUG("Reading workspace state from {}", WorkspaceRoot); std::filesystem::path ConfigPath = WorkspaceRoot / WorkspaceConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = WorkspaceSharesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); @@ -886,7 +886,7 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work { throw std::invalid_argument(fmt::format("invalid root path '{}' for workspace {}", Configuration.RootPath, Configuration.Id)); } - if (!std::filesystem::is_directory(Configuration.RootPath)) + if (!IsDir(Configuration.RootPath)) { throw std::invalid_argument( fmt::format("workspace root path '{}' does not exist for workspace '{}'", Configuration.RootPath, Configuration.Id)); @@ -965,7 +965,7 @@ Workspaces::AddWorkspaceShare(const LoggerRef& Log, throw std::invalid_argument( fmt::format("workspace share path '{}' is not a sub-path of workspace path '{}'", Configuration.SharePath, WorkspaceRoot)); } - if (!std::filesystem::is_directory(WorkspaceRoot / Configuration.SharePath)) + if (!IsDir(WorkspaceRoot / Configuration.SharePath)) { throw std::invalid_argument( fmt::format("workspace share path '{}' does not exist in workspace path '{}'", Configuration.SharePath, WorkspaceRoot)); @@ -1244,7 +1244,7 @@ Workspaces::FindWorkspaceShare(const Oid& WorkspaceId, const Oid& ShareId, bool const Workspaces::WorkspaceConfiguration& WorkspaceConfig = Workspace->GetConfig(); const Workspaces::WorkspaceShareConfiguration& ShareConfig = Share->GetConfig(); std::filesystem::path FullSharePath = WorkspaceConfig.RootPath / ShareConfig.SharePath; - if (std::filesystem::is_directory(FullSharePath)) + if (IsDir(FullSharePath)) { if (ForceRefresh || !Share->IsInitialized()) { @@ -1306,18 +1306,18 @@ namespace { std::filesystem::path EmptyFolder(RootPath / "empty_folder"); std::filesystem::path FirstFolder(RootPath / "first_folder"); - std::filesystem::create_directory(FirstFolder); + CreateDirectories(FirstFolder); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22))); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122))); std::filesystem::path SecondFolder(RootPath / "second_folder"); - std::filesystem::create_directory(SecondFolder); + CreateDirectories(SecondFolder); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225))); std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second"); - std::filesystem::create_directory(SecondFolderChild); + CreateDirectories(SecondFolderChild); Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622))); for (const auto& It : Result) @@ -1365,13 +1365,13 @@ TEST_CASE("workspaces.scanfolder") Structure->IterateEntries([&](const Oid& Id, const FolderStructure::FileEntry& Entry) { std::filesystem::path AbsPath = RootPath / Entry.RelativePath; - CHECK(std::filesystem::is_regular_file(AbsPath)); - CHECK(std::filesystem::file_size(AbsPath) == Entry.Size); + CHECK(IsFile(AbsPath)); + CHECK(FileSizeFromPath(AbsPath) == Entry.Size); const FolderStructure::FileEntry* FindEntry = Structure->FindEntry(Id); CHECK(FindEntry); std::filesystem::path Path = RootPath / FindEntry->RelativePath; CHECK(AbsPath == Path); - CHECK(std::filesystem::file_size(AbsPath) == FindEntry->Size); + CHECK(FileSizeFromPath(AbsPath) == FindEntry->Size); }); } diff --git a/src/zenutil/cache/rpcrecording.cpp b/src/zenutil/cache/rpcrecording.cpp index 1f951167d..380c182b2 100644 --- a/src/zenutil/cache/rpcrecording.cpp +++ b/src/zenutil/cache/rpcrecording.cpp @@ -46,7 +46,7 @@ struct RecordedRequestsWriter void BeginWrite(const std::filesystem::path& BasePath) { m_BasePath = BasePath; - std::filesystem::create_directories(m_BasePath); + CreateDirectories(m_BasePath); } void EndWrite() @@ -426,7 +426,7 @@ RecordedRequestsSegmentWriter::BeginWrite(const std::filesystem::path& BasePath, m_BasePath = BasePath; m_SegmentIndex = SegmentIndex; m_RequestBaseIndex = RequestBaseIndex; - std::filesystem::create_directories(m_BasePath); + CreateDirectories(m_BasePath); } void @@ -1051,14 +1051,14 @@ public: static bool IsCompatible(const std::filesystem::path& BasePath) { - if (std::filesystem::exists(BasePath / "rpc_recording_info.zcb")) + if (IsFile(BasePath / "rpc_recording_info.zcb")) { return true; } const std::filesystem::path SegmentZero = BasePath / MakeSegmentPath(0); - if (std::filesystem::exists(SegmentZero / "rpc_segment_info.zcb") && std::filesystem::exists(SegmentZero / "index.bin")) + if (IsFile(SegmentZero / "rpc_segment_info.zcb") && IsFile(SegmentZero / "index.bin")) { // top-level metadata is missing, possibly because of premature exit // on the recording side diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index f040e9ece..0fa05194f 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -235,7 +235,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView()); @@ -260,7 +260,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); @@ -346,7 +346,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { BasicFile File(BlockPath, BasicFile::Mode::kRead); IoBuffer Payload; @@ -383,7 +383,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { struct WorkloadData { @@ -461,7 +461,7 @@ public: if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash)) { std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); @@ -493,7 +493,7 @@ public: for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); - if (std::filesystem::is_regular_file(MetaDataFile)) + if (IsFile(MetaDataFile)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); @@ -623,7 +623,7 @@ protected: BuildPartObject.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsBinaryAttachment(); const std::filesystem::path BlockPath = GetBlobPayloadPath(AttachmentHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { NeededAttachments.push_back(AttachmentHash); } diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index b36f11741..c0c2a754a 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -534,7 +534,7 @@ ZenServerEnvironment::CreateNewTestDir() TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1)); std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str(); - ZEN_ASSERT(!std::filesystem::exists(TestPath)); + ZEN_ASSERT(!IsDir(TestPath)); ZEN_INFO("Creating new test dir @ '{}'", TestPath); @@ -568,7 +568,7 @@ ZenServerInstance::~ZenServerInstance() { Shutdown(); std::error_code DummyEc; - std::filesystem::remove(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc); + RemoveFile(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc); } catch (const std::exception& Err) { @@ -1033,7 +1033,7 @@ std::string ZenServerInstance::GetLogOutput() const { std::filesystem::path OutputPath = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); - if (std::filesystem::is_regular_file(OutputPath)) + if (IsFile(OutputPath)) { FileContents Contents = ReadFile(OutputPath); if (!Contents.ErrorCode) @@ -1123,7 +1123,7 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason) OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort); return false; } - if (!std::filesystem::is_directory(Info.DataDir)) + if (!IsDir(Info.DataDir)) { OutReason = fmt::format("data directory ('{}') does not exist", Info.DataDir); return false; diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp index a3cfe9d15..9cec5372a 100644 --- a/src/zenvfs/vfsprovider.cpp +++ b/src/zenvfs/vfsprovider.cpp @@ -373,13 +373,12 @@ VfsProvider::Initialize() std::filesystem::path ManifestPath = Root / ".zen_vfs"; bool HaveManifest = false; - if (std::filesystem::exists(Root)) + if (IsFile(Root)) + { + throw std::runtime_error("specified VFS root exists but is not a directory"); + } + if (IsDir(Root)) { - if (!std::filesystem::is_directory(Root)) - { - throw std::runtime_error("specified VFS root exists but is not a directory"); - } - std::error_code Ec; m_RootPath = WideToUtf8(CanonicalPath(Root, Ec).c_str()); @@ -388,7 +387,7 @@ VfsProvider::Initialize() throw std::system_error(Ec); } - if (std::filesystem::exists(ManifestPath)) + if (IsFile(ManifestPath)) { FileContents ManifestData = zen::ReadFile(ManifestPath); CbObject Manifest = LoadCompactBinaryObject(ManifestData.Flatten()); |