Add separate blockstore.h/.cpp

3 files changed, 378 insertions, 0 deletions

diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
new file mode 100644
index 000000000..1aebae0f3
--- /dev/null
+++ b/zenstore/blockstore.cpp
@@ -0,0 +1,273 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "compactcas.h"
+
+#include <zenstore/blockstore.h>
+
+#include <zencore/filesystem.h>
+
+#if ZEN_WITH_TESTS
+#	include <zencore/compactbinarybuilder.h>
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#	include <algorithm>
+#	include <random>
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+BlockStoreFile::BlockStoreFile(const std::filesystem::path& BlockPath) : m_Path(BlockPath)
+{
+}
+
+BlockStoreFile::~BlockStoreFile()
+{
+	RwLock::ExclusiveLockScope _(m_OpenLock);
+	m_IoBuffer = IoBuffer();
+	m_File.Detach();
+}
+
+const std::filesystem::path&
+BlockStoreFile::GetPath() const
+{
+	return m_Path;
+}
+
+void
+BlockStoreFile::InternalOpen()
+{
+	if (m_File.Handle())
+	{
+		return;
+	}
+	m_File.Open(m_Path, false);
+	void* FileHandle = m_File.Handle();
+	m_IoBuffer		 = IoBuffer(IoBuffer::File, FileHandle, 0, m_File.FileSize());
+}
+
+void
+BlockStoreFile::Open()
+{
+	RwLock::ExclusiveLockScope _(m_OpenLock);
+	InternalOpen();
+}
+
+void
+BlockStoreFile::Create(uint64_t InitialSize)
+{
+	RwLock::ExclusiveLockScope _(m_OpenLock);
+
+	auto ParentPath = m_Path.parent_path();
+	if (!std::filesystem::is_directory(ParentPath))
+	{
+		CreateDirectories(ParentPath);
+	}
+
+	m_File.Open(m_Path, true);
+	if (InitialSize > 0)
+	{
+		m_File.SetFileSize(InitialSize);
+	}
+	void* FileHandle = m_File.Handle();
+	m_IoBuffer		 = IoBuffer(IoBuffer::File, FileHandle, 0, InitialSize);
+}
+
+uint64_t
+BlockStoreFile::FileSize()
+{
+	RwLock::SharedLockScope _(m_OpenLock);
+	return m_File.FileSize();
+}
+
+void
+BlockStoreFile::MarkAsDeleteOnClose(std::error_code& Ec)
+{
+	RwLock::ExclusiveLockScope _(m_OpenLock);
+	if (m_File.Handle())
+	{
+		m_File.MarkAsDeleteOnClose(Ec);
+		return;
+	}
+	if (std::filesystem::is_regular_file(m_Path))
+	{
+		Ec.clear();
+		std::filesystem::remove(m_Path, Ec);
+	}
+}
+
+IoBuffer
+BlockStoreFile::GetChunk(uint64_t Offset, uint64_t Size)
+{
+	InternalOpen();
+	return IoBuffer(m_IoBuffer, Offset, Size);
+}
+
+void
+BlockStoreFile::Read(void* Data, uint64_t Size, uint64_t FileOffset)
+{
+	RwLock::SharedLockScope _(m_OpenLock);
+	m_File.Read(Data, Size, FileOffset);
+}
+
+void
+BlockStoreFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
+{
+	RwLock::SharedLockScope _(m_OpenLock);
+	m_File.Write(Data, Size, FileOffset);
+}
+
+void
+BlockStoreFile::Flush()
+{
+	RwLock::ExclusiveLockScope _(m_OpenLock);
+	if (!m_File.Handle())
+	{
+		return;
+	}
+	m_File.Flush();
+}
+
+void
+BlockStoreFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun)
+{
+	RwLock::SharedLockScope _(m_OpenLock);
+	m_File.StreamByteRange(FileOffset, Size, std::move(ChunkFun));
+}
+
+#if ZEN_WITH_TESTS
+
+static bool
+operator==(const BlockStoreLocation& Lhs, const BlockStoreLocation& Rhs)
+{
+	return Lhs.BlockIndex == Rhs.BlockIndex && Lhs.Offset == Rhs.Offset && Lhs.Size == Rhs.Size;
+}
+
+TEST_CASE("blockstore.blockstoredisklocation")
+{
+	BlockStoreLocation Zero = BlockStoreLocation{.BlockIndex = 0, .Offset = 0, .Size = 0};
+	CHECK(Zero == BlockStoreDiskLocation(Zero, 4).Get(4));
+
+	BlockStoreLocation MaxBlockIndex = BlockStoreLocation{.BlockIndex = BlockStoreDiskLocation::MaxBlockIndex, .Offset = 0, .Size = 0};
+	CHECK(MaxBlockIndex == BlockStoreDiskLocation(MaxBlockIndex, 4).Get(4));
+
+	BlockStoreLocation MaxOffset = BlockStoreLocation{.BlockIndex = 0, .Offset = BlockStoreDiskLocation::MaxOffset * 4, .Size = 0};
+	CHECK(MaxOffset == BlockStoreDiskLocation(MaxOffset, 4).Get(4));
+
+	BlockStoreLocation MaxSize = BlockStoreLocation{.BlockIndex = 0, .Offset = 0, .Size = std::numeric_limits<uint32_t>::max()};
+	CHECK(MaxSize == BlockStoreDiskLocation(MaxSize, 4).Get(4));
+
+	BlockStoreLocation MaxBlockIndexAndOffset =
+		BlockStoreLocation{.BlockIndex = BlockStoreDiskLocation::MaxBlockIndex, .Offset = BlockStoreDiskLocation::MaxOffset * 4, .Size = 0};
+	CHECK(MaxBlockIndexAndOffset == BlockStoreDiskLocation(MaxBlockIndexAndOffset, 4).Get(4));
+
+	BlockStoreLocation MaxAll = BlockStoreLocation{.BlockIndex = BlockStoreDiskLocation::MaxBlockIndex,
+												   .Offset	   = BlockStoreDiskLocation::MaxOffset * 4,
+												   .Size	   = std::numeric_limits<uint32_t>::max()};
+	CHECK(MaxAll == BlockStoreDiskLocation(MaxAll, 4).Get(4));
+
+	BlockStoreLocation MaxAll4096 = BlockStoreLocation{.BlockIndex = BlockStoreDiskLocation::MaxBlockIndex,
+													   .Offset	   = BlockStoreDiskLocation::MaxOffset * 4096,
+													   .Size	   = std::numeric_limits<uint32_t>::max()};
+	CHECK(MaxAll4096 == BlockStoreDiskLocation(MaxAll4096, 4096).Get(4096));
+
+	BlockStoreLocation Middle = BlockStoreLocation{.BlockIndex = (BlockStoreDiskLocation::MaxBlockIndex) / 2,
+												   .Offset	   = ((BlockStoreDiskLocation::MaxOffset) / 2) * 4,
+												   .Size	   = std::numeric_limits<uint32_t>::max() / 2};
+	CHECK(Middle == BlockStoreDiskLocation(Middle, 4).Get(4));
+}
+
+TEST_CASE("blockstore.blockfile")
+{
+	ScopedTemporaryDirectory TempDir;
+	auto					 RootDirectory = TempDir.Path() / "blocks";
+	CreateDirectories(RootDirectory);
+
+	{
+		BlockStoreFile File1(RootDirectory / "1");
+		File1.Create(16384);
+		CHECK(File1.FileSize() == 16384);
+		File1.Write("data", 5, 0);
+		auto DataChunk = File1.GetChunk(0, 5);
+		File1.Write("boop", 5, 5);
+		auto BoopChunk = File1.GetChunk(5, 5);
+		auto Data	   = static_cast<const char*>(DataChunk.GetData());
+		CHECK(std::string(Data) == "data");
+		auto Boop = static_cast<const char*>(BoopChunk.GetData());
+		CHECK(std::string(Boop) == "boop");
+		File1.Flush();
+	}
+	{
+		BlockStoreFile File1(RootDirectory / "1");
+		File1.Open();
+
+		char DataRaw[5];
+		File1.Read(DataRaw, 5, 0);
+		CHECK(std::string(DataRaw) == "data");
+		auto DataChunk = File1.GetChunk(0, 5);
+
+		char BoopRaw[5];
+		File1.Read(BoopRaw, 5, 5);
+		CHECK(std::string(BoopRaw) == "boop");
+
+		auto BoopChunk = File1.GetChunk(5, 5);
+		auto Data	   = static_cast<const char*>(DataChunk.GetData());
+		CHECK(std::string(Data) == "data");
+		auto Boop = static_cast<const char*>(BoopChunk.GetData());
+		CHECK(std::string(Boop) == "boop");
+	}
+
+	{
+		IoBuffer DataChunk;
+		IoBuffer BoopChunk;
+
+		{
+			BlockStoreFile File1(RootDirectory / "1");
+			File1.Open();
+			DataChunk = File1.GetChunk(0, 5);
+			BoopChunk = File1.GetChunk(5, 5);
+		}
+
+		CHECK(std::filesystem::exists(RootDirectory / "1"));
+
+		auto Data = static_cast<const char*>(DataChunk.GetData());
+		CHECK(std::string(Data) == "data");
+		auto Boop = static_cast<const char*>(BoopChunk.GetData());
+		CHECK(std::string(Boop) == "boop");
+	}
+	CHECK(std::filesystem::exists(RootDirectory / "1"));
+
+	{
+		IoBuffer DataChunk;
+		IoBuffer BoopChunk;
+
+		{
+			BlockStoreFile File1(RootDirectory / "1");
+			File1.Open();
+			std::error_code Ec;
+			File1.MarkAsDeleteOnClose(Ec);
+			auto M = Ec.message();
+			CHECK(!Ec);
+			DataChunk = File1.GetChunk(0, 5);
+			BoopChunk = File1.GetChunk(5, 5);
+		}
+
+		auto Data = static_cast<const char*>(DataChunk.GetData());
+		CHECK(std::string(Data) == "data");
+		auto Boop = static_cast<const char*>(BoopChunk.GetData());
+		CHECK(std::string(Boop) == "boop");
+	}
+	CHECK(!std::filesystem::exists(RootDirectory / "1"));
+}
+
+#endif
+
+void
+blockstore_forcelink()
+{
+}
+
+}  // namespace zen
diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h
new file mode 100644
index 000000000..29f520722
--- /dev/null
+++ b/zenstore/include/zenstore/blockstore.h
@@ -0,0 +1,103 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+struct BlockStoreLocation
+{
+	uint32_t BlockIndex;
+	uint64_t Offset;
+	uint64_t Size;
+};
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct BlockStoreDiskLocation
+{
+	constexpr static uint32_t MaxBlockIndexBits = 20;
+	constexpr static uint32_t MaxOffsetBits		= 28;
+	constexpr static uint32_t MaxBlockIndex		= (1ul << BlockStoreDiskLocation::MaxBlockIndexBits) - 1ul;
+	constexpr static uint32_t MaxOffset			= (1ul << BlockStoreDiskLocation::MaxOffsetBits) - 1ul;
+
+	BlockStoreDiskLocation(const BlockStoreLocation& Location, uint64_t OffsetAlignement)
+	{
+		Init(Location.BlockIndex, Location.Offset / OffsetAlignement, Location.Size);
+	}
+
+	BlockStoreDiskLocation() = default;
+
+	inline BlockStoreLocation Get(uint64_t OffsetAlignement) const
+	{
+		uint64_t PackedOffset = 0;
+		memcpy(&PackedOffset, &m_Offset, sizeof m_Offset);
+		return {.BlockIndex = static_cast<std::uint32_t>(PackedOffset >> MaxOffsetBits),
+				.Offset		= (PackedOffset & MaxOffset) * OffsetAlignement,
+				.Size		= GetSize()};
+	}
+
+	inline uint32_t GetBlockIndex() const
+	{
+		uint64_t PackedOffset = 0;
+		memcpy(&PackedOffset, &m_Offset, sizeof m_Offset);
+		return static_cast<std::uint32_t>(PackedOffset >> MaxOffsetBits);
+	}
+
+	inline uint64_t GetOffset() const
+	{
+		uint64_t PackedOffset = 0;
+		memcpy(&PackedOffset, &m_Offset, sizeof m_Offset);
+		return PackedOffset & MaxOffset;
+	}
+
+	inline uint64_t GetSize() const { return m_Size; }
+
+private:
+	inline void Init(uint32_t BlockIndex, uint64_t Offset, uint64_t Size)
+	{
+		ZEN_ASSERT(BlockIndex <= MaxBlockIndex);
+		ZEN_ASSERT(Offset <= MaxOffset);
+		ZEN_ASSERT(Size <= std::numeric_limits<std::uint32_t>::max());
+
+		m_Size				  = static_cast<uint32_t>(Size);
+		uint64_t PackedOffset = (static_cast<uint64_t>(BlockIndex) << MaxOffsetBits) + Offset;
+		memcpy(&m_Offset[0], &PackedOffset, sizeof m_Offset);
+	}
+
+	uint32_t m_Size;
+	uint8_t	 m_Offset[6];
+};
+
+#pragma pack(pop)
+
+struct BlockStoreFile
+{
+	explicit BlockStoreFile(const std::filesystem::path& BlockPath);
+	~BlockStoreFile();
+	const std::filesystem::path& GetPath() const;
+	void						 Open();
+	void						 Create(uint64_t InitialSize);
+	void						 MarkAsDeleteOnClose(std::error_code& Ec);
+	uint64_t					 FileSize();
+	IoBuffer					 GetChunk(uint64_t Offset, uint64_t Size);
+	void						 Read(void* Data, uint64_t Size, uint64_t FileOffset);
+	void						 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
+	void						 Flush();
+	void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+
+private:
+	void						InternalOpen();
+	const std::filesystem::path m_Path;
+	RwLock						m_OpenLock;
+	BasicFile					m_File;
+	IoBuffer					m_IoBuffer;
+};
+
+void blockstore_forcelink();
+
+}  // namespace zen
diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp
index dbb3dbbf7..5f40b7f60 100644
--- a/zenstore/zenstore.cpp
+++ b/zenstore/zenstore.cpp
@@ -3,6 +3,7 @@
 #include "zenstore/zenstore.h"
 
 #include <zenstore/basicfile.h>
+#include <zenstore/blockstore.h>
 #include <zenstore/cas.h>
 #include <zenstore/gc.h>
 #include "compactcas.h"
@@ -16,6 +17,7 @@ zenstore_forcelinktests()
 	basicfile_forcelink();
 	CAS_forcelink();
 	filecas_forcelink();
+	blockstore_forcelink();
 	compactcas_forcelink();
 	gc_forcelink();
 }