diff options
| -rw-r--r-- | CHANGELOG.md | 3 | ||||
| -rw-r--r-- | src/zenremotestore/chunking/chunkingcontroller.cpp | 56 | ||||
| -rw-r--r-- | src/zenremotestore/include/zenremotestore/chunking/chunkingcontroller.h | 6 |
3 files changed, 50 insertions, 15 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index d41e1e045..018af1b33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ - Improvement: Better progress reporting for `zen builds upload` and `zen builds download` when using `--log-progress` - Improvement: Validation of files is now part of the "Writing Chunk" progress avoiding stall of feedback at end of the phase - Improvement: Cleaned up http response error logging +- Improvement: Exclude .sym and .psym files from chunking and compress them as individual chunks +- Improvement: Exclude .ogg and .jpg files from chunking and compress them as individual chunks +- Improvement: Exclude various source code file extensions from chunking and compress them as individual chunks - Bugfix: Add quotes around messages when using `--log-progress` with `zen builds` commands ## 5.7.6 diff --git a/src/zenremotestore/chunking/chunkingcontroller.cpp b/src/zenremotestore/chunking/chunkingcontroller.cpp index 49332c2ce..cc20446ea 100644 --- a/src/zenremotestore/chunking/chunkingcontroller.cpp +++ b/src/zenremotestore/chunking/chunkingcontroller.cpp @@ -9,6 +9,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> +#include <tsl/robin_set.h> ZEN_THIRD_PARTY_INCLUDES_END namespace zen { @@ -81,7 +82,14 @@ namespace { class BasicChunkingController : public ChunkingController { public: - BasicChunkingController(const BasicChunkingControllerSettings& Settings) : m_Settings(Settings) {} + BasicChunkingController(const BasicChunkingControllerSettings& Settings) : m_Settings(Settings) + { + m_ExcludeExtensionHashes.reserve(Settings.ExcludeExtensions.size()); + for (const std::string& Extension : Settings.ExcludeExtensions) + { + m_ExcludeExtensionHashes.insert(HashStringAsLowerDjb2(Extension)); + } + } BasicChunkingController(CbObjectView Parameters) : m_Settings(ReadSettings(Parameters)) {} @@ -92,11 +100,15 @@ public: std::atomic<bool>& AbortFlag) const override { ZEN_TRACE_CPU("BasicChunkingController::ProcessFile"); - const bool ExcludeFromChunking = - std::find(m_Settings.ExcludeExtensions.begin(), m_Settings.ExcludeExtensions.end(), InputPath.extension()) != - m_Settings.ExcludeExtensions.end(); + if (RawSize < m_Settings.ChunkFileSizeLimit) + { + return false; + } + + const uint32_t ExtensionHash = HashStringAsLowerDjb2(InputPath.extension().string()); + const bool ExcludeFromChunking = m_ExcludeExtensionHashes.contains(ExtensionHash); - if (ExcludeFromChunking || (RawSize < m_Settings.ChunkFileSizeLimit)) + if (ExcludeFromChunking) { return false; } @@ -151,12 +163,26 @@ private: } const BasicChunkingControllerSettings m_Settings; + tsl::robin_set<uint32_t> m_ExcludeExtensionHashes; }; class ChunkingControllerWithFixedChunking : public ChunkingController { public: - ChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Settings) : m_Settings(Settings) {} + ChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Settings) : m_Settings(Settings) + { + m_ExcludeExtensionHashes.reserve(Settings.ExcludeExtensions.size()); + for (const std::string& Extension : Settings.ExcludeExtensions) + { + m_ExcludeExtensionHashes.insert(HashStringAsLowerDjb2(Extension)); + } + + m_FixedChunkingExtensionHashes.reserve(Settings.FixedChunkingExtensions.size()); + for (const std::string& Extension : Settings.FixedChunkingExtensions) + { + m_FixedChunkingExtensionHashes.insert(HashStringAsLowerDjb2(Extension)); + } + } ChunkingControllerWithFixedChunking(CbObjectView Parameters) : m_Settings(ReadSettings(Parameters)) {} @@ -167,18 +193,20 @@ public: std::atomic<bool>& AbortFlag) const override { ZEN_TRACE_CPU("ChunkingControllerWithFixedChunking::ProcessFile"); - const bool ExcludeFromChunking = - std::find(m_Settings.ExcludeExtensions.begin(), m_Settings.ExcludeExtensions.end(), InputPath.extension()) != - m_Settings.ExcludeExtensions.end(); + if (RawSize < m_Settings.ChunkFileSizeLimit) + { + return false; + } + + const uint32_t ExtensionHash = HashStringAsLowerDjb2(InputPath.extension().string()); + const bool ExcludeFromChunking = m_ExcludeExtensionHashes.contains(ExtensionHash); - if (ExcludeFromChunking || (RawSize < m_Settings.ChunkFileSizeLimit)) + if (ExcludeFromChunking) { return false; } - const bool FixedChunkingExtension = - std::find(m_Settings.FixedChunkingExtensions.begin(), m_Settings.FixedChunkingExtensions.end(), InputPath.extension()) != - m_Settings.FixedChunkingExtensions.end(); + const bool FixedChunkingExtension = m_FixedChunkingExtensionHashes.contains(ExtensionHash); if (FixedChunkingExtension) { @@ -318,6 +346,8 @@ private: } const ChunkingControllerWithFixedChunkingSettings m_Settings; + tsl::robin_set<uint32_t> m_FixedChunkingExtensionHashes; + tsl::robin_set<uint32_t> m_ExcludeExtensionHashes; }; std::unique_ptr<ChunkingController> diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkingcontroller.h b/src/zenremotestore/include/zenremotestore/chunking/chunkingcontroller.h index 2d1ba36aa..b49d76a09 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkingcontroller.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkingcontroller.h @@ -11,8 +11,10 @@ namespace zen { -const std::vector<std::string> DefaultChunkingExcludeExtensions = - {".exe", ".dll", ".pdb", ".self", ".mp4", ".zip", ".7z", ".bzip", ".rar", ".gzip"}; +const std::vector<std::string> DefaultChunkingExcludeExtensions = { + ".exe", ".dll", ".pdb", ".self", ".mp4", ".zip", ".7z", ".bzip", ".rar", ".gzip", ".sym", ".psym", ".txt", ".ini", ".json", + ".verse", ".versemodule", ".jpg", ".c", ".h", ".cpp", ".cxx", ".c++", ".cc", ".hpp", ".hxx", ".h++", ".py", ".ogg", ".plist"}; + const std::vector<std::string> DefaultFixedChunkingExtensions = {".apk", ".nsp", ".xvc", ".pkg", ".dmg", ".ipa"}; const bool DefaultChunkingExcludeElfFiles = true; const bool DefaultChunkingExcludeMachOFiles = true; |