aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2021-05-13 18:22:09 +0200
committerStefan Boberg <[email protected]>2021-05-13 18:22:09 +0200
commit3672bfa998f932bb306e88bf1202a548bb463f24 (patch)
tree4b6b90bef9a42f9cc1a34660d248d985d66177e5
parentImplemented move-in-place for large CAS payloads (diff)
downloadzen-3672bfa998f932bb306e88bf1202a548bb463f24.tar.xz
zen-3672bfa998f932bb306e88bf1202a548bb463f24.zip
Added logic to guard against accidental file move in file CAS fast path
Now verifies the chunk is referencing an entire file before moving it into place
-rw-r--r--zencore/include/zencore/iobuffer.h18
-rw-r--r--zencore/iobuffer.cpp7
-rw-r--r--zenstore/filecas.cpp5
3 files changed, 25 insertions, 5 deletions
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 7e7182997..7cf497527 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -90,6 +90,7 @@ public:
}
inline bool IsImmutable() const { return !(m_Flags & kIsMutable); }
+ inline bool IsWholeFile() const { return !!(m_Flags & kIsWholeFile); }
inline bool IsNull() const { return m_DataBytes == 0; }
inline IoBufferExtendedCore* ExtendedCore();
@@ -123,6 +124,18 @@ public:
}
}
+ inline void SetIsWholeFile(bool NewState)
+ {
+ if (NewState)
+ {
+ m_Flags |= kIsWholeFile;
+ }
+ else
+ {
+ m_Flags |= ~kIsWholeFile;
+ }
+ }
+
inline uint32_t GetRefCount() const { return m_RefCount; }
protected:
@@ -139,6 +152,7 @@ protected:
kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore
kIsMaterialized = 1 << 3, // Data pointers are valid
kLowLevelAlloc = 1 << 4, // Using direct memory allocation
+ kIsWholeFile = 1 << 5, // References an entire file
};
void* AllocateBuffer(size_t InSize, size_t Alignment);
@@ -262,6 +276,7 @@ public:
inline operator bool() const { return !m_Core->IsNull(); }
ZENCORE_API void MakeOwned() { return m_Core->MakeOwned(); }
inline bool IsOwned() const { return m_Core->IsOwned(); }
+ inline bool IsWholeFile() const { return m_Core->IsWholeFile(); }
const void* Data() const { return m_Core->DataPointer(); }
const size_t Size() const { return m_Core->DataBytes(); }
ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const;
@@ -272,6 +287,7 @@ private:
IoBuffer(IoBufferCore* Core) : m_Core(Core) {}
friend class SharedBuffer;
+ friend class IoBufferBuilder;
};
class IoBufferBuilder
@@ -281,8 +297,6 @@ public:
ZENCORE_API static IoBuffer MakeFromTemporaryFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull);
inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
-
-private:
};
void iobuffer_forcelink();
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index e74287c48..d32a30358 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -339,7 +339,7 @@ IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName, uint64_t Offset,
// We need to open with DELETE since this is used for the case
// when a file has been written to a staging directory, and is going
- // to be moved in place.
+ // to be moved in place
HRESULT hRes = DataFile.Create(FileName, GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING);
@@ -361,7 +361,10 @@ IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName, uint64_t Offset,
}
}
- return IoBuffer(IoBuffer::File, DataFile.Detach(), Offset, Size);
+ IoBuffer Iob(IoBuffer::File, DataFile.Detach(), Offset, Size);
+ Iob.m_Core->SetIsWholeFile(true);
+
+ return Iob;
}
return {};
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 0011ddec6..1e9b50e63 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -63,8 +63,11 @@ FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHas
CasStore::InsertResult
FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
{
+ // File-based chunks have special case handling whereby we move the file into
+ // place in the file store directory, thus avoiding unnecessary copying
+
IoBufferFileReference FileRef;
- if (Chunk.GetFileReference(/* out */ FileRef))
+ if (Chunk.IsWholeFile() && Chunk.GetFileReference(/* out */ FileRef))
{
size_t Shard2len = 0;
ExtendableWideStringBuilder<128> ShardedPath;