aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--zencore/compactbinarypackage.cpp5
-rw-r--r--zencore/include/zencore/iobuffer.h3
-rw-r--r--zencore/include/zencore/sharedbuffer.h3
-rw-r--r--zencore/iobuffer.cpp35
-rw-r--r--zenserver/projectstore.cpp6
-rw-r--r--zenstore/filecas.cpp108
6 files changed, 155 insertions, 5 deletions
diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index f84137ff6..262b6ecba 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -54,7 +54,10 @@ CbAttachment::CbAttachment(SharedBuffer InBuffer, const IoHash* const InHash) :
Hash = *InHash;
if (Buffer.GetSize())
{
- ZEN_ASSERT_SLOW(Hash == IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize()));
+ // This is disabled for now as it forces disk-based attachments to get mapped which
+ // then prevents us from making them delete themselves on close
+
+ // ZEN_ASSERT_SLOW(Hash == IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize()));
}
else
{
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index f38af3c27..7e7182997 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -269,6 +269,8 @@ public:
private:
RefPtr<IoBufferCore> m_Core = new IoBufferCore;
+ IoBuffer(IoBufferCore* Core) : m_Core(Core) {}
+
friend class SharedBuffer;
};
@@ -276,6 +278,7 @@ class IoBufferBuilder
{
public:
ZENCORE_API static IoBuffer MakeFromFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
+ ZENCORE_API static IoBuffer MakeFromTemporaryFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull);
inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h
index 7f9c61adb..cdcf66da4 100644
--- a/zencore/include/zencore/sharedbuffer.h
+++ b/zencore/include/zencore/sharedbuffer.h
@@ -101,7 +101,8 @@ public:
}
}
- operator MemoryView() const { return GetView(); }
+ operator MemoryView() const { return GetView(); }
+ inline IoBuffer AsIoBuffer() const { return IoBuffer(m_Buffer); }
SharedBuffer& operator=(UniqueBuffer&& Rhs)
{
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index ec5d599b4..e74287c48 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -332,6 +332,41 @@ IoBufferBuilder::MakeFromFile(const wchar_t* FileName, uint64_t Offset, uint64_t
return {};
}
+IoBuffer
+IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName, uint64_t Offset, uint64_t Size)
+{
+ CAtlFile DataFile;
+
+ // We need to open with DELETE since this is used for the case
+ // when a file has been written to a staging directory, and is going
+ // to be moved in place.
+
+ HRESULT hRes = DataFile.Create(FileName, GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING);
+
+ if (SUCCEEDED(hRes))
+ {
+ ULONGLONG FileSize;
+ DataFile.GetSize(FileSize);
+
+ if (Size == ~0ull)
+ {
+ Size = FileSize;
+ }
+ else
+ {
+ // Clamp size
+ if ((Offset + Size) > FileSize)
+ {
+ Size = FileSize - Offset;
+ }
+ }
+
+ return IoBuffer(IoBuffer::File, DataFile.Detach(), Offset, Size);
+ }
+
+ return {};
+}
+
//////////////////////////////////////////////////////////////////////////
void
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index cb6be69e8..dcad483a6 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -477,8 +477,8 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage)
for (const auto& Attach : Attachments)
{
- SharedBuffer BinaryView = Attach.AsBinaryView();
- m_CasStore.InsertChunk(BinaryView.GetData(), BinaryView.GetSize(), Attach.GetHash());
+ IoBuffer AttachmentData = Attach.AsBinaryView().AsIoBuffer();
+ m_CasStore.InsertChunk(AttachmentData, Attach.GetHash());
}
return RegisterOplogEntry(Core, OpEntry, kUpdateNewEntry);
@@ -1060,7 +1060,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
std::filesystem::path AttachmentPath = Log.TempPath() / Hash.ToHexString();
- if (IoBuffer Data = IoBufferBuilder::MakeFromFile(AttachmentPath.native().c_str()))
+ if (IoBuffer Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath.native().c_str()))
{
return SharedBuffer(std::move(Data));
}
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 84a06c3be..0011ddec6 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -3,11 +3,14 @@
#include "FileCas.h"
#include <zencore/except.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
#include <zencore/memory.h>
#include <zencore/string.h>
#include <zencore/thread.h>
#include <zencore/uid.h>
+#include <spdlog/spdlog.h>
#include <gsl/gsl-lite.hpp>
#include <functional>
@@ -25,6 +28,8 @@ struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax erro
namespace zen {
+using namespace fmt::literals;
+
WideStringBuilderBase&
FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len)
{
@@ -58,6 +63,109 @@ FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHas
CasStore::InsertResult
FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
{
+ IoBufferFileReference FileRef;
+ if (Chunk.GetFileReference(/* out */ FileRef))
+ {
+ size_t Shard2len = 0;
+ ExtendableWideStringBuilder<128> ShardedPath;
+ ShardedPath.Append(m_Config.RootDirectory.c_str());
+ ShardedPath.Append(std::filesystem::path::preferred_separator);
+ MakeShardedPath(ShardedPath, ChunkHash, /* out */ Shard2len);
+
+ auto DeletePayloadFileOnClose = [&] {
+ FILE_DISPOSITION_INFO Fdi{};
+ Fdi.DeleteFile = TRUE;
+ BOOL Success = SetFileInformationByHandle(FileRef.FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);
+
+ if (!Success)
+ {
+ spdlog::warn("Failed to flag temporary payload file for deletion: '{}'", PathFromHandle(FileRef.FileHandle));
+ }
+ };
+
+ // See if file already exists
+ //
+ // Future improvement: maintain Bloom filter to avoid expensive file system probes?
+
+ {
+ CAtlFile PayloadFile;
+
+ HRESULT hRes = PayloadFile.Create(ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
+
+ if (SUCCEEDED(hRes))
+ {
+ // If we succeeded in opening the file then we don't need to do anything else because it already exists and should contain
+ // the content we were about to insert
+
+ // We do need to ensure the file goes away on close, however
+
+ DeletePayloadFileOnClose();
+
+ return CasStore::InsertResult{.New = false};
+ }
+ }
+
+ std::filesystem::path FullPath(ShardedPath.c_str());
+
+ std::filesystem::path FilePath = FullPath.parent_path();
+ std::wstring FileName = FullPath.native();
+
+ const DWORD BufferSize = sizeof(FILE_RENAME_INFO) + gsl::narrow<DWORD>(FileName.size() * sizeof(WCHAR));
+ FILE_RENAME_INFO* RenameInfo = reinterpret_cast<FILE_RENAME_INFO*>(Memory::Alloc(BufferSize));
+ memset(RenameInfo, 0, BufferSize);
+
+ RenameInfo->ReplaceIfExists = FALSE;
+ RenameInfo->FileNameLength = gsl::narrow<DWORD>(FileName.size());
+ memcpy(RenameInfo->FileName, FileName.c_str(), FileName.size() * sizeof(WCHAR));
+ RenameInfo->FileName[FileName.size()] = 0;
+
+ // Try to move file into place
+
+ BOOL Success = SetFileInformationByHandle(FileRef.FileHandle, FileRenameInfo, RenameInfo, BufferSize);
+
+ if (!Success)
+ {
+ CAtlFile DirHandle;
+
+ auto InternalCreateDirectoryHandle = [&] {
+ return DirHandle.Create(FilePath.c_str(),
+ GENERIC_READ | GENERIC_WRITE,
+ FILE_SHARE_READ | FILE_SHARE_WRITE,
+ OPEN_EXISTING,
+ FILE_FLAG_BACKUP_SEMANTICS);
+ };
+
+ HRESULT hRes = InternalCreateDirectoryHandle();
+
+ if (FAILED(hRes))
+ {
+ zen::CreateDirectories(FilePath.c_str());
+
+ hRes = InternalCreateDirectoryHandle();
+ }
+
+ if (FAILED(hRes))
+ {
+ throw WindowsException(hRes, "Failed to open shard directory '{}'"_format(FilePath));
+ }
+
+ // Retry
+
+ Success = SetFileInformationByHandle(FileRef.FileHandle, FileRenameInfo, RenameInfo, BufferSize);
+ }
+
+ Memory::Free(RenameInfo);
+
+ if (Success)
+ {
+ return CasStore::InsertResult{.New = true};
+ }
+
+ spdlog::warn("rename of CAS payload file failed, falling back to regular write for {}", ChunkHash);
+
+ DeletePayloadFileOnClose();
+ }
+
return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash);
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-25 19:03:45 +0000