moved source directories into `/src` (#264)

* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees

1 files changed, 102 insertions, 0 deletions

diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h
new file mode 100644
index 000000000..420b3a634
--- /dev/null
+++ b/src/zenstore/filecas.h
@@ -0,0 +1,102 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/filesystem.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/thread.h>
+#include <zenstore/caslog.h>
+#include <zenstore/gc.h>
+
+#include "cas.h"
+
+#include <atomic>
+#include <functional>
+
+namespace spdlog {
+class logger;
+}
+
+namespace zen {
+
+class BasicFile;
+
+/** CAS storage strategy using a file-per-chunk storage strategy
+ */
+
+struct FileCasStrategy final : public GcStorage
+{
+	FileCasStrategy(GcManager& Gc);
+	~FileCasStrategy();
+
+	void				   Initialize(const std::filesystem::path& RootDirectory, bool IsNewStore);
+	CasStore::InsertResult InsertChunk(IoBuffer				Chunk,
+									   const IoHash&		ChunkHash,
+									   CasStore::InsertMode Mode = CasStore::InsertMode::kMayBeMovedInPlace);
+	IoBuffer			   FindChunk(const IoHash& ChunkHash);
+	bool				   HaveChunk(const IoHash& ChunkHash);
+	void				   FilterChunks(HashKeySet& InOutChunks);
+	void				   Flush();
+	void				   Scrub(ScrubContext& Ctx);
+	virtual void		   CollectGarbage(GcContext& GcCtx) override;
+	virtual GcStorageSize  StorageSize() const override { return {.DiskSize = m_TotalSize.load(std::memory_order::relaxed)}; }
+
+private:
+	void	 MakeIndexSnapshot();
+	uint64_t ReadIndexFile();
+	uint64_t ReadLog(uint64_t LogPosition);
+
+	struct IndexEntry
+	{
+		uint64_t Size = 0;
+	};
+	using IndexMap = tsl::robin_map<IoHash, IndexEntry, IoHash::Hasher>;
+
+	CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
+
+	std::filesystem::path m_RootDirectory;
+	RwLock				  m_Lock;
+	IndexMap			  m_Index;
+	RwLock				  m_ShardLocks[256];  // TODO: these should be spaced out so they don't share cache lines
+	spdlog::logger&		  m_Log;
+	spdlog::logger&		  Log() { return m_Log; }
+	std::atomic_uint64_t  m_TotalSize{};
+	bool				  m_IsInitialized = false;
+
+	struct FileCasIndexEntry
+	{
+		static const uint32_t kTombStone = 0x0000'0001;
+
+		bool IsFlagSet(const uint32_t Flag) const { return (Flags & kTombStone) == Flag; }
+
+		IoHash	 Key;
+		uint32_t Flags = 0;
+		uint64_t Size  = 0;
+	};
+	static bool											   ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutReason);
+	static std::vector<FileCasStrategy::FileCasIndexEntry> ScanFolderForCasFiles(const std::filesystem::path& RootDir);
+
+	static_assert(sizeof(FileCasIndexEntry) == 32);
+
+	TCasLogFile<FileCasIndexEntry> m_CasLog;
+	uint64_t					   m_LogFlushPosition = 0;
+
+	inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardLocks[Hash.Hash[19]]; }
+	void		   IterateChunks(std::function<void(const IoHash& Hash, IoBuffer&& Payload)>&& Callback);
+	void		   DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec);
+
+	struct ShardingHelper
+	{
+		ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash);
+
+		size_t					   Shard2len = 0;
+		ExtendablePathBuilder<128> ShardedPath;
+	};
+};
+
+void filecas_forcelink();
+
+}  // namespace zen