optimizations (#200)

* Use direct file read and direct buffer allocation for small IoBuffer materalization
* Reduce range of materialized data in CompositeBuffer reading
CompressedBuffer header reading often only need a small part and not the whole file
* reduce lock contention in IoBuffer::Materialize
* Reduce parsing of compressed headers
Validate header type at decompression
* faster CreateDirectories - start from leaf going up and recurse back
* optimized BufferHeader::IsValid
* Add ValidateCompressedHeader to use when we don't need the actual compressed data
Validate that we always get compressed data in CidStore::AddChunk
* changelog

1 files changed, 2 insertions, 19 deletions

diff --git a/zen/cmds/exportproject.cpp b/zen/cmds/exportproject.cpp
index 5d8c7d536..36d37bb60 100644
--- a/zen/cmds/exportproject.cpp
+++ b/zen/cmds/exportproject.cpp
@@ -22,23 +22,6 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <gsl/gsl-lite.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
-namespace {
-
-void
-EnsureDirectoryExists(const std::filesystem::path& Path)
-{
-	while (!std::filesystem::is_directory(Path))
-	{
-		if (Path.has_parent_path())
-		{
-			EnsureDirectoryExists(Path.parent_path());
-		}
-		std::filesystem::create_directory(Path);
-	}
-}
-
-}  // namespace
-
 ExportProjectCommand::ExportProjectCommand()
 {
 	m_Options.add_options()("h,help", "Print help");
@@ -147,7 +130,7 @@ ExportProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
 			return 1;
 		}
 		zen::IoBuffer CompressedPayload(zen::IoBuffer::Wrap, Response.text.data(), Response.text.size());
-		zen::IoBuffer Payload = zen::CompressedBuffer::FromCompressed(zen::SharedBuffer(CompressedPayload)).Decompress().AsIoBuffer();
+		zen::IoBuffer Payload = zen::CompressedBuffer::FromCompressedNoValidate(std::move(CompressedPayload)).Decompress().AsIoBuffer();
 
 		OplogResponses.emplace_back(zen::ParsePackageMessage(Payload));
 		zen::CbPackage& ResponsePackage = OplogResponses.back();
@@ -291,7 +274,7 @@ ExportProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
 			JobCount.fetch_add(1);
 			WorkerPool.ScheduleWork([this, AttachmentBody, &Chunk, &JobCount]() {
 				std::filesystem::path Path = GetLargeChunkPath(m_TargetPath, Chunk.ChunkHash);
-				EnsureDirectoryExists(Path.parent_path());
+				zen::CreateDirectories(Path.parent_path());
 				zen::BasicFile ChunkFile;
 				ChunkFile.Open(Path, zen::BasicFile::Mode::kTruncate);
 				uint64_t Offset = 0;