From 5bc5b0dd59c0f02afe553e5074dfe57951b19044 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 25 Feb 2025 15:48:43 +0100 Subject: improvements and infrastructure for upcoming builds api command line (#284) * add modification tick to filesystem traversal * add ShowDetails option to ProgressBar * log callstack if we terminate process * handle chunking if MaxSize > 1MB * BasicFile write helpers and WriteToTempFile simplifications * bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data revert of earlier optimization * faster compress/decompress for large disk-based files * enable progress feedback in IoHash::HashBuffer * add payload validation in HttpClient::Get * fix range requests (range is including end byte) * remove BuildPartId for blob/block related operations in builds api --- src/zencore/filesystem.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'src/zencore/filesystem.cpp') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index b8c35212f..5716d1255 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -683,7 +683,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop { } - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override { std::error_code Ec; const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec); @@ -1236,7 +1236,11 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr } else { - Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart, gsl::narrow(DirInfo->FileAttributes)); + Visitor.VisitFile(RootDir, + FileName, + DirInfo->EndOfFile.QuadPart, + gsl::narrow(DirInfo->FileAttributes), + (uint64_t)DirInfo->LastWriteTime.QuadPart); } const uint64_t NextOffset = DirInfo->NextEntryOffset; @@ -1285,7 +1289,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr } else if (S_ISREG(Stat.st_mode)) { - Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow(Stat.st_mode)); + Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow(Stat.st_mode), gsl::narrow(Stat.st_mtime)); } else { @@ -1544,7 +1548,8 @@ GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, - uint32_t NativeModeOrAttributes) override + uint32_t NativeModeOrAttributes, + uint64_t NativeModificationTick) override { if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)) { @@ -1557,6 +1562,10 @@ GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags { Content.FileAttributes.push_back(NativeModeOrAttributes); } + if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeModificationTick)) + { + Content.FileModificationTicks.push_back(NativeModificationTick); + } } } @@ -1612,7 +1621,8 @@ GetDirectoryContent(const std::filesystem::path& RootDir, virtual void VisitFile(const std::filesystem::path&, const path_view& File, uint64_t FileSize, - uint32_t NativeModeOrAttributes) override + uint32_t NativeModeOrAttributes, + uint64_t NativeModificationTick) override { if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)) { @@ -1625,6 +1635,10 @@ GetDirectoryContent(const std::filesystem::path& RootDir, { Content.FileAttributes.push_back(NativeModeOrAttributes); } + if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeModificationTick)) + { + Content.FileModificationTicks.push_back(NativeModificationTick); + } } } @@ -1928,7 +1942,7 @@ TEST_CASE("filesystem") // Traversal struct : public FileSystemTraversal::TreeVisitor { - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t, uint64_t) override { bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected); } -- cgit v1.2.3 From 7d8fe45af3b49d800f84f0ddce051c0b3b2e837d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Feb 2025 15:10:14 +0100 Subject: builds upload command (#278) - Feature: **EXPERIMENTAL** New `zen builds` command to list, upload and download folders to Cloud Build API - `builds list` list available builds (**INCOMPLETE - FILTERING MISSING**) - `builds upload` upload a folder to Cloud Build API - `--local-path` source folder to upload - `--create-build` creates a new parent build object (using the object id), if omitted a parent build must exist and `--build-id` must be given - `--build-id` an Oid in hex form for the Build identifier to use - omit to have the id auto generated - `--build-part-id` and Oid in hex form for the Build Part identifier for the folder - omit to have the id auto generated - `--build-part-name` name of the build part - if omitted the name of the leaf folder name give in `--local-path` - `--metadata-path` path to a json formatted file with meta data information about the build. Meta-data must be provided if `--create-build` is set - `--metadata` key-value pairs separated by ';' with build meta data for the build. (key1=value1;key2=value2). Meta-data must be provided if `--create-build` is set - `--clean` ignore any existing blocks of chunk data and upload a fresh set of blocks - `--allow-multipart` enable usage of multi-part http upload requests - `--manifest-path` path to text file listing files to include in upload. Exclude to upload everything in `--local-path` - `builds download` download a folder from Cloud Build API (**INCOMPLETE - WILL WIPE UNTRACKED DATA FROM TARGET FOLDER**) - `--local-path` target folder to download to - `--build-id` an Oid in hex form for the Build identifier to use - `--build-part-id` a comma separated list of Oid in hex for the build part identifier(s) to download - mutually exclusive to `--build-part-name` - `--build-part-name` a comma separated list of names for the build part(s) to download - if omitted the name of the leaf folder name give in `--local-path` - `--clean` deletes all data in target folder before downloading (NON-CLEAN IS NOT IMPLEMENTED YET) - `--allow-multipart` enable usage of multi-part http download reqeusts - `builds diff` download a folder from Cloud Build API - `--local-path` target folder to download to - `--compare-path` folder to compare target with - `--only-chunked` compare only files that would be chunked - `builds fetch-blob` fetch and validate a blob from remote store - `--build-id` an Oid in hex form for the Build identifier to use - `--blob-hash` an IoHash in hex form identifying the blob to download - `builds validate part` fetch a build part and validate all referenced attachments - `--build-id` an Oid in hex form for the Build identifier to use - `--build-part-id` an Oid in hex for the build part identifier to validate - mutually exclusive to `--build-part-name` - `--build-part-name` a name for the build part to validate - mutually exclusive to `--build-part-id` - `builds test` a series of operation that uploads, downloads and test various aspects of incremental operations - `--local-path` source folder to upload - Options for Cloud Build API remote store (`list`, `upload`, `download`, `fetch-blob`, `validate-part`) - `--url` Cloud Builds URL - `--assume-http2` assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake - `--namespace` Builds Storage namespace - `--bucket` Builds Storage bucket - Authentication options for Cloud Build API - Auth token - `--access-token` http auth Cloud Storage access token - `--access-token-env` name of environment variable that holds the Http auth Cloud Storage access token - `--access-token-path` path to json file that holds the Http auth Cloud Storage access token - OpenId authentication - `--openid-provider-name` Open ID provider name - `--openid-provider-url` Open ID provider url - `--openid-client-id`Open ID client id - `--openid-refresh-token` Open ID refresh token - `--encryption-aes-key` 256 bit AES encryption key for storing OpenID credentials - `--encryption-aes-iv` 128 bit AES encryption initialization vector for storing OpenID credentials - OAuth authentication - `--oauth-url` OAuth provier url - `--oauth-clientid` OAuth client id - `--oauth-clientsecret` OAuth client secret - Options for file based remote store used for for testing purposes (`list`, `upload`, `download`, `fetch-blob`, `validate-part`, `test`) - `--storage-path` path to folder to store builds data - `--json-metadata` enable json output in store for all compact binary objects (off by default) - Output options for all builds commands - `--plain-progress` use plain line-by-line progress output - `--verbose` --- src/zencore/filesystem.cpp | 140 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) (limited to 'src/zencore/filesystem.cpp') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 5716d1255..8279fb952 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -1469,6 +1469,36 @@ GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec) return 0; } +uint64_t +GetModificationTickFromPath(const std::filesystem::path& Filename) +{ + // PathFromHandle + void* Handle; +#if ZEN_PLATFORM_WINDOWS + Handle = CreateFileW(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); +#else + int Fd = open(Filename.c_str(), O_RDONLY | O_CLOEXEC); + if (Fd <= 9) + { + ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); + } + Handle = (void*)uintptr_t(Fd); + auto _ = MakeGuard([Handle]() { close(int(uintptr_t(Handle))); }); +#endif + std::error_code Ec; + uint64_t ModificatonTick = GetModificationTickFromHandle(Handle, Ec); + if (Ec) + { + ThrowSystemError(Ec.value(), Ec.message()); + } + return ModificatonTick; +} + std::filesystem::path GetRunningExecutablePath() { @@ -1895,6 +1925,116 @@ PickDefaultSystemRootDirectory() #endif // ZEN_PLATFORM_WINDOWS } +#if ZEN_PLATFORM_WINDOWS + +uint32_t +GetFileAttributes(const std::filesystem::path& Filename) +{ + DWORD Attributes = ::GetFileAttributes(Filename.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + ThrowLastError(fmt::format("failed to get attributes of file {}", Filename)); + } + return (uint32_t)Attributes; +} + +void +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +{ + if (::SetFileAttributes(Filename.native().c_str(), Attributes) == 0) + { + ThrowLastError(fmt::format("failed to set attributes of file {}", Filename)); + } +} + +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + +uint32_t +GetFileMode(const std::filesystem::path& Filename) +{ + struct stat Stat; + int err = stat(Filename.native().c_str(), &Stat); + if (err) + { + ThrowLastError(fmt::format("Failed to get mode of file {}", Filename)); + } + return (uint32_t)Stat.st_mode; +} + +void +SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes) +{ + int err = chmod(Filename.native().c_str(), (mode_t)Attributes); + if (err) + { + ThrowLastError(fmt::format("Failed to set mode of file {}", Filename)); + } +} + +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + +#if ZEN_PLATFORM_WINDOWS +const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; +#else +const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; +#endif // ZEN_PLATFORM_WINDOWS + +const uint32_t FileModeWriteEnableFlags = 0222; + +bool +IsFileAttributeReadOnly(uint32_t FileAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; +#else + return (FileAttributes & 0x00000001) != 0; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +IsFileModeReadOnly(uint32_t FileMode) +{ + return (FileMode & FileModeWriteEnableFlags) == 0; +} + +uint32_t +MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +{ + return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); +} + +uint32_t +MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) +{ + return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); +} + +bool +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +{ +#if ZEN_PLATFORM_WINDOWS + uint32_t CurrentAttributes = GetFileAttributes(Filename); + uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); + if (CurrentAttributes != NewAttributes) + { + SetFileAttributes(Filename, NewAttributes); + return true; + } +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + uint32_t CurrentMode = GetFileMode(Filename); + uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); + if (CurrentMode != NewMode) + { + SetFileMode(Filename, NewMode); + return true; + } +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + return false; +} + ////////////////////////////////////////////////////////////////////////// // // Testing related code follows... -- cgit v1.2.3 From 19b3c492dcc0fc3f8879ecb60124ca64dea9b7ef Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 1 Mar 2025 10:10:53 +0100 Subject: builds download incremental (#290) * incremental download * merge rebuild state and output state building * fix writing when > 1 zero size file --- src/zencore/filesystem.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/zencore/filesystem.cpp') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 8279fb952..85feab2f7 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -531,7 +531,10 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) } void -CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options, std::error_code& OutErrorCode) +CopyFile(const std::filesystem::path& FromPath, + const std::filesystem::path& ToPath, + const CopyFileOptions& Options, + std::error_code& OutErrorCode) { OutErrorCode.clear(); @@ -544,7 +547,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop } bool -CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) +CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options) { bool Success = false; -- cgit v1.2.3 From 26f718b04bc0fba3831124d9dec705c2febd426e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Mar 2025 11:19:06 +0100 Subject: fix mac/linux builds command (#303) * fix linux/mac version of GetModificationTickFromPath and CopyFile --- src/zencore/filesystem.cpp | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) (limited to 'src/zencore/filesystem.cpp') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 85feab2f7..9f3f4f7fc 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -590,7 +590,7 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP ScopedFd $From = {FromFd}; // To file - int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666); + int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); if (ToFd < 0) { ThrowLastError(fmt::format("failed to create file {}", ToPath)); @@ -598,9 +598,14 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP fchmod(ToFd, 0666); ScopedFd $To = {ToFd}; + struct stat Stat; + fstat(FromFd, &Stat); + + size_t FileSizeBytes = Stat.st_size; + // Copy impl - static const size_t BufferSize = 64 << 10; - void* Buffer = malloc(BufferSize); + const size_t BufferSize = Min(FileSizeBytes, 64u << 10); + void* Buffer = malloc(BufferSize); while (true) { int BytesRead = read(FromFd, Buffer, BufferSize); @@ -610,7 +615,7 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP break; } - if (write(ToFd, Buffer, BytesRead) != BufferSize) + if (write(ToFd, Buffer, BytesRead) != BytesRead) { Success = false; break; @@ -621,7 +626,7 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP if (!Success) { - ThrowLastError("file copy failed"sv); + ThrowLastError(fmt::format("file copy from {} to {} failed", FromPath, ToPath)); } return true; @@ -1483,16 +1488,7 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) { ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); } - auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); -#else - int Fd = open(Filename.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd <= 9) - { - ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); - } - Handle = (void*)uintptr_t(Fd); - auto _ = MakeGuard([Handle]() { close(int(uintptr_t(Handle))); }); -#endif + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); std::error_code Ec; uint64_t ModificatonTick = GetModificationTickFromHandle(Handle, Ec); if (Ec) @@ -1500,6 +1496,15 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) ThrowSystemError(Ec.value(), Ec.message()); } return ModificatonTick; +#else + struct stat Stat; + int err = stat(Filename.native().c_str(), &Stat); + if (err) + { + ThrowLastError(fmt::format("Failed to get mode of file {}", Filename)); + } + return gsl::narrow(Stat.st_mtime); +#endif } std::filesystem::path -- cgit v1.2.3 From 908f99b749fbbf9754f9485d680914792034334c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Mar 2025 18:58:24 +0100 Subject: fix quoted command lines arguments (#306) Handling of quotes and quotes with leading backslash for command line parsing - UE-231677 --- src/zencore/filesystem.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/zencore/filesystem.cpp') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 9f3f4f7fc..05e2bf049 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2043,6 +2043,19 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) return false; } +std::filesystem::path +StringToPath(const std::string_view& Path) +{ + if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"') + { + return std::filesystem::path(Path.substr(1, Path.length() - 2)).make_preferred(); + } + else + { + return std::filesystem::path(Path).make_preferred(); + } +} + ////////////////////////////////////////////////////////////////////////// // // Testing related code follows... -- cgit v1.2.3