diff options
| author | Stefan Boberg <[email protected]> | 2026-03-16 10:16:46 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2026-03-16 10:16:46 +0100 |
| commit | e5b97e8e769108ba45d6a064a46f892b68f1e950 (patch) | |
| tree | c0d9abb1191b0caf3d35fdaa1c1f267c17a0bf85 | |
| parent | add buildid updates to oplog and builds test scripts (#838) (diff) | |
| download | zen-e5b97e8e769108ba45d6a064a46f892b68f1e950.tar.xz zen-e5b97e8e769108ba45d6a064a46f892b68f1e950.zip | |
block/file cloning support for macOS / Linux (#786)
- Add block cloning (copy-on-write) support for Linux and macOS to complement the existing Windows (ReFS) implementation
- **Linux**: `TryCloneFile` via `FICLONE` ioctl, `CloneQueryInterface` with range cloning via `FICLONERANGE` (Btrfs/XFS)
- **macOS**: `TryCloneFile` via `clonefile()` syscall (APFS), `SupportsBlockRefCounting` via `VOL_CAP_INT_CLONE`. `CloneQueryInterface` is not implemented as macOS lacks a sub-file range clone API
- Promote `ScopedFd` to file scope for broader use in filesystem code
- Add test scripts for block cloning validation on Linux (Btrfs via loopback) and macOS (APFS)
- Also added test script for testing on Windows (ReFS)
| -rw-r--r-- | CHANGELOG.md | 2 | ||||
| -rwxr-xr-x | scripts/test_scripts/block-clone-test-mac.sh | 43 | ||||
| -rw-r--r-- | scripts/test_scripts/block-clone-test-windows.ps1 | 145 | ||||
| -rwxr-xr-x | scripts/test_scripts/block-clone-test.sh | 143 | ||||
| -rw-r--r-- | src/zencore/filesystem.cpp | 587 | ||||
| -rw-r--r-- | src/zencore/include/zencore/filesystem.h | 8 |
6 files changed, 856 insertions, 72 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 6844e5e4f..cd0f85c98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ - Feature: Added `zen down --all` flag to shut down all running zenserver instances - Feature: Added `xmake kill` task to terminate all running zenserver instances - Feature: Support `ZEN_MALLOC` environment variable for default allocator selection; default allocator switched to rpmalloc +- Improvement: Added support for CoW block-cloning (used by build download) on Linux (tested with: btrfs/ XFS) +- Improvement: Added full-file CoW copying on macOS (APFS) - Improvement: Updated asio to 1.38.0 - Improvement: Updated fmt to 1.12.1 - Bugfix: Fixed sentry-native build to allow LTO on Windows diff --git a/scripts/test_scripts/block-clone-test-mac.sh b/scripts/test_scripts/block-clone-test-mac.sh new file mode 100755 index 000000000..a3d3ca4d3 --- /dev/null +++ b/scripts/test_scripts/block-clone-test-mac.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Test block-clone functionality on macOS (APFS). +# +# APFS is the default filesystem on modern Macs and natively supports +# clonefile(), so no special setup is needed — just run the tests. +# +# Usage: +# ./scripts/test_scripts/block-clone-test-mac.sh [path-to-zencore-test] +# +# If no path is given, defaults to build/macosx/<arch>/debug/zencore-test +# relative to the repository root. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +ARCH="$(uname -m)" +TEST_BINARY="${1:-$REPO_ROOT/build/macosx/$ARCH/debug/zencore-test}" + +if [ ! -x "$TEST_BINARY" ]; then + echo "error: test binary not found or not executable: $TEST_BINARY" >&2 + echo "hint: build with 'xmake config -m debug && xmake build zencore-test'" >&2 + exit 1 +fi + +# Verify we're on APFS +BINARY_DIR="$(dirname "$TEST_BINARY")" +FS_TYPE="$(diskutil info "$(df "$BINARY_DIR" | tail -1 | awk '{print $1}')" 2>/dev/null | grep "Type (Bundle)" | awk '{print $NF}' || true)" + +if [ "$FS_TYPE" != "apfs" ]; then + echo "warning: filesystem does not appear to be APFS (got: ${FS_TYPE:-unknown}), clone tests may skip" >&2 +fi + +TEST_CASES="TryCloneFile,CopyFile.Clone,SupportsBlockRefCounting,CloneQueryInterface" + +echo "Running block-clone tests ..." +echo "---" +"$TEST_BINARY" \ + --test-suite="core.filesystem" \ + --test-case="$TEST_CASES" +echo "---" +echo "All block-clone tests passed." diff --git a/scripts/test_scripts/block-clone-test-windows.ps1 b/scripts/test_scripts/block-clone-test-windows.ps1 new file mode 100644 index 000000000..df24831a4 --- /dev/null +++ b/scripts/test_scripts/block-clone-test-windows.ps1 @@ -0,0 +1,145 @@ +# Test block-clone functionality on a temporary ReFS VHD. +# +# Requires: +# - Administrator privileges +# - Windows Server, or Windows 10/11 Pro for Workstations (ReFS support) +# - Hyper-V PowerShell module (for New-VHD), or diskpart fallback +# +# Usage: +# # From an elevated PowerShell prompt: +# .\scripts\test_scripts\block-clone-test-windows.ps1 [-TestBinary <path>] +# +# If -TestBinary is not given, defaults to build\windows\x64\debug\zencore-test.exe +# relative to the repository root. + +param( + [string]$TestBinary = "" +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Definition +$RepoRoot = (Resolve-Path "$ScriptDir\..\..").Path + +if (-not $TestBinary) { + $TestBinary = Join-Path $RepoRoot "build\windows\x64\debug\zencore-test.exe" +} + +$ImageSizeMB = 2048 +$TestCases = "TryCloneFile,CopyFile.Clone,SupportsBlockRefCounting,CloneQueryInterface" + +$VhdPath = "" +$MountLetter = "" + +function Cleanup { + $ErrorActionPreference = "SilentlyContinue" + + if ($MountLetter) { + Write-Host "Dismounting VHD ..." + Dismount-VHD -Path $VhdPath -ErrorAction SilentlyContinue + } + if ($VhdPath -and (Test-Path $VhdPath)) { + Remove-Item -Force $VhdPath -ErrorAction SilentlyContinue + } +} + +trap { + Cleanup + throw $_ +} + +# --- Preflight checks --- + +$IsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator) +if (-not $IsAdmin) { + Write-Error "This script must be run as Administrator (for VHD mount/format)." + exit 1 +} + +if (-not (Test-Path $TestBinary)) { + Write-Error "Test binary not found: $TestBinary`nHint: build with 'xmake config -m debug && xmake build zencore-test'" + exit 1 +} + +# Check that ReFS formatting is available +$RefsAvailable = $true +try { + # A quick check: on non-Server/Workstation SKUs, Format-Volume -FileSystem ReFS will fail + $OsCaption = (Get-CimInstance Win32_OperatingSystem).Caption + if ($OsCaption -notmatch "Server|Workstation|Enterprise") { + Write-Warning "ReFS may not be available on this Windows edition: $OsCaption" + Write-Warning "Continuing anyway — format step will fail if unsupported." + } +} catch { + # Non-fatal, just proceed +} + +# --- Create and mount ReFS VHD --- + +$VhdPath = Join-Path $env:TEMP "refs-clone-test-$([guid]::NewGuid().ToString('N').Substring(0,8)).vhdx" + +Write-Host "Creating ${ImageSizeMB}MB VHDX at $VhdPath ..." + +try { + # Prefer Hyper-V cmdlet if available + New-VHD -Path $VhdPath -SizeBytes ($ImageSizeMB * 1MB) -Fixed | Out-Null +} catch { + # Fallback to diskpart + Write-Host "New-VHD not available, falling back to diskpart ..." + $DiskpartScript = @" +create vdisk file="$VhdPath" maximum=$ImageSizeMB type=fixed +"@ + $DiskpartScript | diskpart | Out-Null +} + +Write-Host "Mounting and initializing VHD ..." + +Mount-VHD -Path $VhdPath +$Disk = Get-VHD -Path $VhdPath | Get-Disk + +# Suppress Explorer's auto-open / "format disk?" prompts for the raw partition +Stop-Service ShellHWDetection -ErrorAction SilentlyContinue + +try { + Initialize-Disk -Number $Disk.Number -PartitionStyle GPT -ErrorAction SilentlyContinue + $Partition = New-Partition -DiskNumber $Disk.Number -UseMaximumSize -AssignDriveLetter + $MountLetter = $Partition.DriveLetter + + Write-Host "Formatting ${MountLetter}: as ReFS with integrity disabled ..." + Format-Volume -DriveLetter $MountLetter -FileSystem ReFS -NewFileSystemLabel "CloneTest" -Confirm:$false | Out-Null + + # Disable integrity streams (required for block cloning to work on ReFS) + Set-FileIntegrity "${MountLetter}:\" -Enable $false -ErrorAction SilentlyContinue +} finally { + Start-Service ShellHWDetection -ErrorAction SilentlyContinue +} + +$MountRoot = "${MountLetter}:\" + +# --- Copy test binary and run --- + +Write-Host "Copying test binary to ReFS volume ..." +Copy-Item $TestBinary "$MountRoot\zencore-test.exe" + +Write-Host "Running block-clone tests ..." +Write-Host "---" + +$proc = Start-Process -FilePath "$MountRoot\zencore-test.exe" ` + -ArgumentList "--test-suite=core.filesystem", "--test-case=$TestCases" ` + -NoNewWindow -Wait -PassThru + +Write-Host "---" + +if ($proc.ExitCode -ne 0) { + Write-Error "Tests failed with exit code $($proc.ExitCode)" + Cleanup + exit $proc.ExitCode +} + +Write-Host "ReFS: all block-clone tests passed." + +# --- Cleanup --- + +Cleanup +Write-Host "Done." diff --git a/scripts/test_scripts/block-clone-test.sh b/scripts/test_scripts/block-clone-test.sh new file mode 100755 index 000000000..7c6bf5605 --- /dev/null +++ b/scripts/test_scripts/block-clone-test.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Test block-clone functionality on temporary Btrfs and XFS loopback filesystems. +# +# Requires: root/sudo, btrfs-progs (mkfs.btrfs), xfsprogs (mkfs.xfs) +# +# Usage: +# sudo ./scripts/test_scripts/block-clone-test.sh [path-to-zencore-test] +# +# If no path is given, defaults to build/linux/x86_64/debug/zencore-test +# relative to the repository root. +# +# Options: +# --btrfs-only Only test Btrfs +# --xfs-only Only test XFS + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +TEST_BINARY="" +RUN_BTRFS=true +RUN_XFS=true + +for arg in "$@"; do + case "$arg" in + --btrfs-only) RUN_XFS=false ;; + --xfs-only) RUN_BTRFS=false ;; + *) TEST_BINARY="$arg" ;; + esac +done + +TEST_BINARY="${TEST_BINARY:-$REPO_ROOT/build/linux/x86_64/debug/zencore-test}" +IMAGE_SIZE="512M" +TEST_CASES="TryCloneFile,CopyFile.Clone,SupportsBlockRefCounting,CloneQueryInterface" + +# Track all temp files for cleanup +CLEANUP_MOUNTS=() +CLEANUP_DIRS=() +CLEANUP_FILES=() + +cleanup() { + local exit_code=$? + set +e + + for mnt in "${CLEANUP_MOUNTS[@]}"; do + if mountpoint -q "$mnt" 2>/dev/null; then + umount "$mnt" + fi + done + for dir in "${CLEANUP_DIRS[@]}"; do + [ -d "$dir" ] && rmdir "$dir" + done + for f in "${CLEANUP_FILES[@]}"; do + [ -f "$f" ] && rm -f "$f" + done + + if [ $exit_code -ne 0 ]; then + echo "FAILED (exit code $exit_code)" + fi + exit $exit_code +} +trap cleanup EXIT + +# --- Preflight checks --- + +if [ "$(id -u)" -ne 0 ]; then + echo "error: this script must be run as root (for mount/umount)" >&2 + exit 1 +fi + +if [ ! -x "$TEST_BINARY" ]; then + echo "error: test binary not found or not executable: $TEST_BINARY" >&2 + echo "hint: build with 'xmake config -m debug && xmake build zencore-test'" >&2 + exit 1 +fi + +if $RUN_BTRFS && ! command -v mkfs.btrfs &>/dev/null; then + echo "warning: mkfs.btrfs not found — install btrfs-progs to test Btrfs, skipping" >&2 + RUN_BTRFS=false +fi + +if $RUN_XFS && ! command -v mkfs.xfs &>/dev/null; then + echo "warning: mkfs.xfs not found — install xfsprogs to test XFS, skipping" >&2 + RUN_XFS=false +fi + +if ! $RUN_BTRFS && ! $RUN_XFS; then + echo "error: no filesystems to test" >&2 + exit 1 +fi + +# --- Helper to create, mount, and run tests on a loopback filesystem --- + +run_tests_on_fs() { + local fs_type="$1" + local mkfs_cmd="$2" + + echo "" + echo "========================================" + echo " Testing block-clone on $fs_type" + echo "========================================" + + local image_path mount_path + image_path="$(mktemp "/tmp/${fs_type}-clone-test-XXXXXX.img")" + mount_path="$(mktemp -d "/tmp/${fs_type}-clone-mount-XXXXXX")" + CLEANUP_FILES+=("$image_path") + CLEANUP_DIRS+=("$mount_path") + CLEANUP_MOUNTS+=("$mount_path") + + echo "Creating ${IMAGE_SIZE} ${fs_type} image at ${image_path} ..." + truncate -s "$IMAGE_SIZE" "$image_path" + $mkfs_cmd "$image_path" + + echo "Mounting at ${mount_path} ..." + mount -o loop "$image_path" "$mount_path" + chmod 777 "$mount_path" + + echo "Copying test binary ..." + cp "$TEST_BINARY" "$mount_path/zencore-test" + chmod +x "$mount_path/zencore-test" + + echo "Running tests ..." + echo "---" + "$mount_path/zencore-test" \ + --test-suite="core.filesystem" \ + --test-case="$TEST_CASES" + echo "---" + echo "$fs_type: all block-clone tests passed." +} + +# --- Run --- + +if $RUN_BTRFS; then + run_tests_on_fs "btrfs" "mkfs.btrfs -q" +fi + +if $RUN_XFS; then + run_tests_on_fs "xfs" "mkfs.xfs -q -m reflink=1" +fi + +echo "" +echo "All block-clone tests passed." diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 8ed63565c..e557b376c 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -32,17 +32,27 @@ ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_PLATFORM_LINUX # include <dirent.h> # include <fcntl.h> +# include <linux/fs.h> +# include <linux/magic.h> +# include <sys/ioctl.h> # include <sys/resource.h> # include <sys/mman.h> # include <sys/stat.h> +# include <sys/vfs.h> # include <pwd.h> # include <unistd.h> +// XFS_SUPER_MAGIC is not always defined in linux/magic.h +# ifndef XFS_SUPER_MAGIC +# define XFS_SUPER_MAGIC 0x58465342 +# endif #endif #if ZEN_PLATFORM_MAC # include <dirent.h> # include <fcntl.h> # include <libproc.h> +# include <sys/attr.h> +# include <sys/clonefile.h> # include <sys/resource.h> # include <sys/mman.h> # include <sys/stat.h> @@ -59,6 +69,53 @@ namespace zen { using namespace std::literals; +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC +struct ScopedFd +{ + int Fd = -1; + + ScopedFd() = default; + explicit ScopedFd(int InFd) : Fd(InFd) {} + + ~ScopedFd() + { + if (Fd >= 0) + { + close(Fd); + } + } + + ScopedFd(const ScopedFd&) = delete; + ScopedFd& operator=(const ScopedFd&) = delete; + + ScopedFd(ScopedFd&& Other) noexcept : Fd(Other.Fd) { Other.Fd = -1; } + + ScopedFd& operator=(ScopedFd&& Other) noexcept + { + if (this != &Other) + { + if (Fd >= 0) + { + close(Fd); + } + Fd = Other.Fd; + Other.Fd = -1; + } + return *this; + } + + // Release ownership of the file descriptor, returning it without closing + int Release() + { + int Result = Fd; + Fd = -1; + return Result; + } + + explicit operator bool() const { return Fd >= 0; } +}; +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + #if ZEN_PLATFORM_WINDOWS static bool @@ -615,6 +672,38 @@ SupportsBlockRefCounting(std::filesystem::path Path) } return true; +#elif ZEN_PLATFORM_LINUX + struct statfs Buf; + if (statfs(Path.c_str(), &Buf) != 0) + { + return false; + } + + // Btrfs and XFS (when formatted with reflink support) support FICLONE + return Buf.f_type == BTRFS_SUPER_MAGIC || Buf.f_type == XFS_SUPER_MAGIC; +#elif ZEN_PLATFORM_MAC + struct attrlist AttrList = {}; + AttrList.bitmapcount = ATTR_BIT_MAP_COUNT; + AttrList.volattr = ATTR_VOL_CAPABILITIES; + + struct + { + uint32_t Length; + vol_capabilities_attr_t Capabilities; + } AttrBuf = {}; + + if (getattrlist(Path.c_str(), &AttrList, &AttrBuf, sizeof(AttrBuf), 0) != 0) + { + return false; + } + + // Check that the VOL_CAP_INT_CLONE bit is both valid and set + if (!(AttrBuf.Capabilities.valid[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_CLONE)) + { + return false; + } + + return !!(AttrBuf.Capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_CLONE); #else ZEN_UNUSED(Path); return false; @@ -768,7 +857,115 @@ private: DWORD m_TargetVolumeSerialNumber; }; -#endif // ZEN_PLATFORM_WINDOWS +#elif ZEN_PLATFORM_LINUX + +class LinuxCloneQueryInterface : public CloneQueryInterface +{ +public: + LinuxCloneQueryInterface(uint64_t AlignmentSize, dev_t TargetDevice) : m_AlignmentSize(AlignmentSize), m_TargetDevice(TargetDevice) {} + + virtual bool CanClone(void* SourceNativeHandle) override + { + int Fd = int(uintptr_t(SourceNativeHandle)); + + struct stat St; + if (fstat(Fd, &St) != 0) + { + return false; + } + + // Source must be on the same filesystem as the target + return St.st_dev == m_TargetDevice; + } + + virtual uint64_t GetClonableRange(uint64_t SourceOffset, + uint64_t TargetOffset, + uint64_t Size, + uint64_t& OutPreBytes, + uint64_t& OutPostBytes) override + { + if (Size < m_AlignmentSize) + { + return 0; + } + + uint64_t PreBytes = (m_AlignmentSize - (SourceOffset % m_AlignmentSize)) % m_AlignmentSize; + uint64_t PostBytes = (SourceOffset + Size) % m_AlignmentSize; + ZEN_ASSERT(Size >= PreBytes + PostBytes); + if (Size - (PreBytes + PostBytes) < m_AlignmentSize) + { + return 0; + } + ZEN_ASSERT((PreBytes < Size && PostBytes < Size && Size >= PreBytes + PostBytes + m_AlignmentSize)); + + const uint64_t DestCloneOffset = TargetOffset + PreBytes; + if (DestCloneOffset % m_AlignmentSize != 0) + { + return 0; + } + + OutPreBytes = PreBytes; + OutPostBytes = PostBytes; + uint64_t CloneSize = Size - (PreBytes + PostBytes); + ZEN_ASSERT(CloneSize % m_AlignmentSize == 0); + return CloneSize; + } + + virtual bool TryClone(void* SourceNativeHandle, + void* TargetNativeHandle, + uint64_t AlignedSourceOffset, + uint64_t AlignedTargetOffset, + uint64_t AlignedSize, + uint64_t TargetFinalSize) override + { + ZEN_ASSERT_SLOW(CanClone(SourceNativeHandle)); + ZEN_ASSERT((AlignedSourceOffset % m_AlignmentSize) == 0); + ZEN_ASSERT((AlignedTargetOffset % m_AlignmentSize) == 0); + ZEN_ASSERT(AlignedSize > 0); + ZEN_ASSERT((AlignedSize % m_AlignmentSize) == 0); + + int SourceFd = int(uintptr_t(SourceNativeHandle)); + int TargetFd = int(uintptr_t(TargetNativeHandle)); + + // Ensure the target file is sized to its final size before cloning + struct stat TargetSt; + if (fstat(TargetFd, &TargetSt) != 0 || uint64_t(TargetSt.st_size) != TargetFinalSize) + { + if (ftruncate(TargetFd, TargetFinalSize) != 0) + { + std::error_code DummyEc; + ZEN_DEBUG("Failed setting final size {} for file {}", TargetFinalSize, PathFromHandle(TargetNativeHandle, DummyEc)); + return false; + } + } + + struct file_clone_range Range = {}; + Range.src_fd = SourceFd; + Range.src_offset = AlignedSourceOffset; + Range.src_length = AlignedSize; + Range.dest_offset = AlignedTargetOffset; + + if (ioctl(TargetFd, FICLONERANGE, &Range) != 0) + { + std::error_code DummyEc; + ZEN_DEBUG("Failed cloning {} bytes from file {} at {} to file {} at {}", + AlignedSize, + PathFromHandle(SourceNativeHandle, DummyEc), + AlignedSourceOffset, + PathFromHandle(TargetNativeHandle, DummyEc), + AlignedTargetOffset); + return false; + } + + return true; + } + +private: + uint64_t m_AlignmentSize; + dev_t m_TargetDevice; +}; + +#endif // ZEN_PLATFORM_WINDOWS / ZEN_PLATFORM_LINUX std::unique_ptr<CloneQueryInterface> GetCloneQueryInterface(const std::filesystem::path& TargetDirectory) @@ -819,7 +1016,30 @@ GetCloneQueryInterface(const std::filesystem::path& TargetDirectory) return std::make_unique<WindowsCloneQueryInterface>(SectorsPerCluster * BytesPerSector, DestVolumeSerialNumber); } } -#else // ZEN_PLATFORM_WINDOWS +#elif ZEN_PLATFORM_LINUX + struct statfs FsBuf; + if (statfs(TargetDirectory.c_str(), &FsBuf) != 0) + { + ZEN_DEBUG("Failed to get filesystem info for path {}", TargetDirectory); + return {}; + } + + // Only Btrfs and XFS support FICLONERANGE + if (FsBuf.f_type != BTRFS_SUPER_MAGIC && FsBuf.f_type != XFS_SUPER_MAGIC) + { + return {}; + } + + struct stat StBuf; + if (stat(TargetDirectory.c_str(), &StBuf) != 0) + { + ZEN_DEBUG("Failed to stat path {}", TargetDirectory); + return {}; + } + + uint64_t AlignmentSize = FsBuf.f_bsize; + return std::make_unique<LinuxCloneQueryInterface>(AlignmentSize, StBuf.st_dev); +#else ZEN_UNUSED(TargetDirectory); #endif // ZEN_PLATFORM_WINDOWS return {}; @@ -1000,40 +1220,44 @@ TryCloneFile(const std::filesystem::path& FromPath, const std::filesystem::path& return TryCloneFile((void*)FromFile.m_Handle, (void*)TargetFile.m_Handle); #elif ZEN_PLATFORM_LINUX -# if 0 - struct ScopedFd - { - ~ScopedFd() { close(Fd); } - int Fd; - }; - // The 'from' file - int FromFd = open(FromPath.c_str(), O_RDONLY|O_CLOEXEC); - if (FromFd < 0) + ScopedFd FromFd(open(FromPath.c_str(), O_RDONLY | O_CLOEXEC)); + if (!FromFd) { return false; } - ScopedFd $From = { FromFd }; + + // Remove any existing target so we can create a fresh clone + unlink(ToPath.c_str()); // The 'to' file - int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC, 0666); - if (ToFd < 0) + ScopedFd ToFd(open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666)); + if (!ToFd) { return false; } - fchmod(ToFd, 0666); - ScopedFd $To = { ToFd }; - ioctl(ToFd, FICLONE, FromFd); + if (ioctl(ToFd.Fd, FICLONE, FromFd.Fd) != 0) + { + // Clone not supported by this filesystem or files are on different volumes. + // Remove the empty target file we created. + ToFd = ScopedFd(); + unlink(ToPath.c_str()); + return false; + } - return false; -# endif // 0 - ZEN_UNUSED(FromPath, ToPath); - return false; + return true; #elif ZEN_PLATFORM_MAC - /* clonefile() syscall if APFS */ - ZEN_UNUSED(FromPath, ToPath); - return false; + // Remove any existing target - clonefile() requires the destination not exist + unlink(ToPath.c_str()); + + if (clonefile(FromPath.c_str(), ToPath.c_str(), CLONE_NOFOLLOW) != 0) + { + // Clone not supported (non-APFS) or files are on different volumes + return false; + } + + return true; #endif // ZEN_PLATFORM_WINDOWS } @@ -1069,9 +1293,7 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP if (Options.MustClone) { -#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - ZEN_ERROR("CloneFile() is not implemented on this platform"); -#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX + ZEN_ERROR("CloneFile() failed for {} -> {}", FromPath, ToPath); return false; } @@ -1084,35 +1306,27 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP &CancelFlag, /* dwCopyFlags */ 0); #else - struct ScopedFd - { - ~ScopedFd() { close(Fd); } - int Fd; - }; - // From file - int FromFd = open(FromPath.c_str(), O_RDONLY | O_CLOEXEC); - if (FromFd < 0) + ScopedFd FromFd(open(FromPath.c_str(), O_RDONLY | O_CLOEXEC)); + if (!FromFd) { ThrowLastError(fmt::format("failed to open file {}", FromPath)); } - ScopedFd $From = {FromFd}; // To file - int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); - if (ToFd < 0) + ScopedFd ToFd(open(ToPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666)); + if (!ToFd) { ThrowLastError(fmt::format("failed to create file {}", ToPath)); } - fchmod(ToFd, 0666); - ScopedFd $To = {ToFd}; + fchmod(ToFd.Fd, 0666); struct stat Stat; - fstat(FromFd, &Stat); + fstat(FromFd.Fd, &Stat); size_t FileSizeBytes = Stat.st_size; - int $Ignore = fchown(ToFd, Stat.st_uid, Stat.st_gid); + int $Ignore = fchown(ToFd.Fd, Stat.st_uid, Stat.st_gid); ZEN_UNUSED($Ignore); // What's the appropriate error handling here? // Copy impl @@ -1120,14 +1334,14 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP void* Buffer = malloc(BufferSize); while (true) { - int BytesRead = read(FromFd, Buffer, BufferSize); + int BytesRead = read(FromFd.Fd, Buffer, BufferSize); if (BytesRead <= 0) { Success = (BytesRead == 0); break; } - if (write(ToFd, Buffer, BytesRead) != BytesRead) + if (write(ToFd.Fd, Buffer, BytesRead) != BytesRead) { Success = false; break; @@ -1371,20 +1585,20 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer } #else - int OpenFlags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC; - int Fd = open(Path.c_str(), OpenFlags, 0666); - if (Fd < 0) + int OpenFlags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC; + ScopedFd OutFd(open(Path.c_str(), OpenFlags, 0666)); + if (!OutFd) { zen::CreateDirectories(Path.parent_path()); - Fd = open(Path.c_str(), OpenFlags, 0666); + OutFd = ScopedFd(open(Path.c_str(), OpenFlags, 0666)); } - if (Fd < 0) + if (!OutFd) { ThrowLastError(fmt::format("File open failed for '{}'", Path)); } - fchmod(Fd, 0666); + fchmod(OutFd.Fd, 0666); #endif // TODO: this should be block-enlightened @@ -1408,9 +1622,9 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else - if (write(Fd, DataPtr, ChunkSize) != int64_t(ChunkSize)) + if (write(OutFd.Fd, DataPtr, ChunkSize) != int64_t(ChunkSize)) { - close(Fd); + OutFd = ScopedFd(); std::error_code DummyEc; RemoveFile(Path, DummyEc); ThrowLastError(fmt::format("File write failed for '{}'", Path)); @@ -1424,8 +1638,6 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer #if ZEN_PLATFORM_WINDOWS Outfile.Close(); -#else - close(Fd); #endif } @@ -1707,8 +1919,8 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi ProcessFunc(ReadBuffer.data(), dwBytesRead); } #else - int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd < 0) + ScopedFd InFd(open(Path.c_str(), O_RDONLY | O_CLOEXEC)); + if (!InFd) { return false; } @@ -1718,7 +1930,7 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi void* Buffer = malloc(ChunkSize); while (true) { - int BytesRead = read(Fd, Buffer, ChunkSize); + int BytesRead = read(InFd.Fd, Buffer, ChunkSize); if (BytesRead < 0) { Success = false; @@ -1734,7 +1946,6 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi } free(Buffer); - close(Fd); if (!Success) { @@ -3123,28 +3334,26 @@ public: ZEN_UNUSED(SystemGlobal); std::string InstanceMapName = fmt::format("/{}", Name); - int Fd = shm_open(InstanceMapName.c_str(), O_RDWR, 0666); - if (Fd < 0) + ScopedFd FdGuard(shm_open(InstanceMapName.c_str(), O_RDWR, 0666)); + if (!FdGuard) { return {}; } - void* hMap = (void*)intptr_t(Fd); struct stat Stat; - fstat(Fd, &Stat); + fstat(FdGuard.Fd, &Stat); if (size_t(Stat.st_size) < Size) { - close(Fd); return {}; } - void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0); + void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, FdGuard.Fd, 0); if (pBuf == MAP_FAILED) { - close(Fd); return {}; } + void* hMap = (void*)intptr_t(FdGuard.Release()); return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)}; #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC } @@ -3199,23 +3408,22 @@ public: ZEN_UNUSED(SystemGlobal); std::string InstanceMapName = fmt::format("/{}", Name); - int Fd = shm_open(InstanceMapName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); - if (Fd < 0) + ScopedFd FdGuard(shm_open(InstanceMapName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666)); + if (!FdGuard) { return {}; } - fchmod(Fd, 0666); - void* hMap = (void*)intptr_t(Fd); + fchmod(FdGuard.Fd, 0666); - int Result = ftruncate(Fd, Size); + int Result = ftruncate(FdGuard.Fd, Size); ZEN_UNUSED(Result); - void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0); + void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, FdGuard.Fd, 0); if (pBuf == MAP_FAILED) { - close(Fd); return {}; } + void* hMap = (void*)intptr_t(FdGuard.Release()); return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)}; #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC } @@ -3590,6 +3798,241 @@ TEST_CASE("RotateDirectories") } } +TEST_CASE("TryCloneFile") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".clone_test"; + CleanDirectory(TestBaseDir, true); + + SUBCASE("clone produces identical content") + { + std::filesystem::path SrcPath = TestBaseDir / "src.bin"; + std::filesystem::path DstPath = TestBaseDir / "dst.bin"; + + // Write source file with known content + const char Content[] = "Hello, clone world! This is test data for TryCloneFile."; + WriteFile(SrcPath, IoBuffer(IoBuffer::Wrap, Content, sizeof(Content))); + CHECK(IsFile(SrcPath)); + + bool Cloned = TryCloneFile(SrcPath, DstPath); + + if (Cloned) + { + CHECK(IsFile(DstPath)); + CHECK_EQ(FileSizeFromPath(DstPath), sizeof(Content)); + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + CHECK_EQ(DstContents.Data[0].GetSize(), sizeof(Content)); + CHECK_EQ(memcmp(DstContents.Data[0].Data(), Content, sizeof(Content)), 0); + } + else + { + // Clone not supported on this filesystem - that's okay, just verify it didn't leave debris + ZEN_INFO("TryCloneFile not supported on this filesystem, skipping content check"); + } + } + + SUBCASE("clone overwrites existing target") + { + std::filesystem::path SrcPath = TestBaseDir / "src_overwrite.bin"; + std::filesystem::path DstPath = TestBaseDir / "dst_overwrite.bin"; + + const char OldContent[] = "old content"; + const char NewContent[] = "new content that is longer than the old one"; + WriteFile(DstPath, IoBuffer(IoBuffer::Wrap, OldContent, sizeof(OldContent))); + WriteFile(SrcPath, IoBuffer(IoBuffer::Wrap, NewContent, sizeof(NewContent))); + + bool Cloned = TryCloneFile(SrcPath, DstPath); + + if (Cloned) + { + CHECK_EQ(FileSizeFromPath(DstPath), sizeof(NewContent)); + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + CHECK_EQ(memcmp(DstContents.Data[0].Data(), NewContent, sizeof(NewContent)), 0); + } + } + + SUBCASE("clone of nonexistent source fails") + { + std::filesystem::path SrcPath = TestBaseDir / "no_such_file.bin"; + std::filesystem::path DstPath = TestBaseDir / "dst_nosrc.bin"; + + CHECK_FALSE(TryCloneFile(SrcPath, DstPath)); + CHECK_FALSE(IsFile(DstPath)); + } + + DeleteDirectories(TestBaseDir); +} + +TEST_CASE("CopyFile.Clone") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".copyfile_clone_test"; + CleanDirectory(TestBaseDir, true); + + const char Content[] = "CopyFile clone test content with some bytes to verify integrity."; + std::filesystem::path SrcPath = TestBaseDir / "src.bin"; + WriteFile(SrcPath, IoBuffer(IoBuffer::Wrap, Content, sizeof(Content))); + + SUBCASE("EnableClone copies file regardless of clone support") + { + std::filesystem::path DstPath = TestBaseDir / "dst_enable.bin"; + + CopyFileOptions Options; + Options.EnableClone = true; + bool Success = CopyFile(SrcPath, DstPath, Options); + CHECK(Success); + CHECK(IsFile(DstPath)); + CHECK_EQ(FileSizeFromPath(DstPath), sizeof(Content)); + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + CHECK_EQ(memcmp(DstContents.Data[0].Data(), Content, sizeof(Content)), 0); + } + + SUBCASE("DisableClone still copies file") + { + std::filesystem::path DstPath = TestBaseDir / "dst_disable.bin"; + + CopyFileOptions Options; + Options.EnableClone = false; + bool Success = CopyFile(SrcPath, DstPath, Options); + CHECK(Success); + CHECK(IsFile(DstPath)); + CHECK_EQ(FileSizeFromPath(DstPath), sizeof(Content)); + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + CHECK_EQ(memcmp(DstContents.Data[0].Data(), Content, sizeof(Content)), 0); + } + + DeleteDirectories(TestBaseDir); +} + +TEST_CASE("SupportsBlockRefCounting") +{ + std::filesystem::path BinDir = GetRunningExecutablePath().parent_path(); + + // Should not crash or throw on a valid path + bool Supported = SupportsBlockRefCounting(BinDir); + ZEN_INFO("SupportsBlockRefCounting({}) = {}", BinDir, Supported); + + // Should return false for nonexistent path + CHECK_FALSE(SupportsBlockRefCounting("/no/such/path/anywhere")); +} + +TEST_CASE("CloneQueryInterface") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".clonequery_test"; + CleanDirectory(TestBaseDir, true); + + auto CloneQuery = GetCloneQueryInterface(TestBaseDir); + + if (CloneQuery) + { + ZEN_INFO("CloneQueryInterface available for {}", TestBaseDir); + + // Write a source file large enough to exercise alignment + const uint64_t FileSize = 256 * 1024; + IoBuffer SrcBuf(FileSize); + { + uint8_t* Ptr = SrcBuf.MutableData<uint8_t>(); + for (uint64_t i = 0; i < FileSize; i++) + { + Ptr[i] = uint8_t(i * 37 + 7); + } + } + + std::filesystem::path SrcPath = TestBaseDir / "clone_src.bin"; + std::filesystem::path DstPath = TestBaseDir / "clone_dst.bin"; + WriteFile(SrcPath, SrcBuf); + + // Open source and target as native handles +# if ZEN_PLATFORM_WINDOWS + windows::Handle SrcHandle(CreateFileW(SrcPath.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr)); + CHECK(SrcHandle != INVALID_HANDLE_VALUE); + void* SrcNativeHandle = (void*)SrcHandle.m_Handle; + + windows::Handle DstHandle( + CreateFileW(DstPath.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, nullptr, OPEN_ALWAYS, 0, nullptr)); + CHECK(DstHandle != INVALID_HANDLE_VALUE); + void* DstNativeHandle = (void*)DstHandle.m_Handle; +# else + ScopedFd SrcFd(open(SrcPath.c_str(), O_RDONLY | O_CLOEXEC)); + CHECK(bool(SrcFd)); + void* SrcNativeHandle = (void*)uintptr_t(SrcFd.Fd); + + ScopedFd DstFd(open(DstPath.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666)); + CHECK(bool(DstFd)); + void* DstNativeHandle = (void*)uintptr_t(DstFd.Fd); +# endif + + SUBCASE("CanClone returns true for same volume") { CHECK(CloneQuery->CanClone(SrcNativeHandle)); } + + SUBCASE("GetClonableRange and TryClone") + { + uint64_t PreBytes = 0; + uint64_t PostBytes = 0; + uint64_t Clonable = CloneQuery->GetClonableRange(0, 0, FileSize, PreBytes, PostBytes); + + if (Clonable > 0) + { + CHECK_EQ(PreBytes, 0); // Offset 0 is always aligned + CHECK(Clonable + PostBytes == FileSize); + + bool Cloned = CloneQuery->TryClone(SrcNativeHandle, DstNativeHandle, 0, 0, Clonable, FileSize); + CHECK(Cloned); + + if (Cloned) + { + // Write the post-alignment tail if any + if (PostBytes > 0) + { + const uint8_t* SrcData = SrcBuf.Data<uint8_t>() + Clonable; +# if ZEN_PLATFORM_WINDOWS + DWORD Written = 0; + OVERLAPPED Ov = {}; + Ov.Offset = (DWORD)(Clonable & 0xFFFFFFFF); + Ov.OffsetHigh = (DWORD)(Clonable >> 32); + ::WriteFile(DstHandle, SrcData, (DWORD)PostBytes, &Written, &Ov); +# else + pwrite(DstFd.Fd, SrcData, PostBytes, Clonable); +# endif + } + + // Close handles before reading back the file for verification +# if ZEN_PLATFORM_WINDOWS + SrcHandle.Close(); + DstHandle.Close(); +# else + SrcFd = ScopedFd(); + DstFd = ScopedFd(); +# endif + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + IoBuffer DstFlat = DstContents.Flatten(); + CHECK_EQ(DstFlat.GetSize(), FileSize); + CHECK_EQ(memcmp(DstFlat.Data(), SrcBuf.Data(), FileSize), 0); + } + } + } + } + else + { + ZEN_INFO("CloneQueryInterface not available for {} (filesystem does not support block cloning)", TestBaseDir); + } + + DeleteDirectories(TestBaseDir); +} + TEST_CASE("SharedMemory") { CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false)); diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 16e2b59f8..6dc159a83 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -187,6 +187,14 @@ void ScanFile(void* NativeHandle, void WriteFile(void* NativeHandle, const void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); void ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); +// Interface for sub-file range cloning on filesystems that support copy-on-write. +// GetCloneQueryInterface() returns nullptr on platforms without range clone support. +// +// Platform capabilities: +// Windows (ReFS) - True CoW range cloning via FSCTL_DUPLICATE_EXTENTS_TO_FILE. +// Linux (Btrfs/XFS) - True CoW range cloning via FICLONERANGE ioctl. +// macOS (APFS) - Not implemented. No sub-file range clone API exists. +// Whole-file CoW cloning is available via TryCloneFile (clonefile syscall). class CloneQueryInterface { public: |