Merged main.

1 files changed, 115 insertions, 14 deletions

diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 2fc968a91..4e4502e6d 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include "FileCas.h"
+#include "filecas.h"
 
 #include <zencore/except.h>
 #include <zencore/filesystem.h>
@@ -28,7 +28,9 @@
 #include <unordered_map>
 
 ZEN_THIRD_PARTY_INCLUDES_START
-#include <atlfile.h>
+#if ZEN_PLATFORM_WINDOWS
+#	include <atlfile.h>
+#endif
 ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
@@ -38,7 +40,6 @@ using namespace fmt::literals;
 FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash)
 {
 	ShardedPath.Append(RootPath.c_str());
-	ShardedPath.Append(std::filesystem::path::preferred_separator);
 
 	ExtendableStringBuilder<64> HashString;
 	ChunkHash.ToHexString(HashString);
@@ -58,13 +59,14 @@ FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& Roo
 	// would probably be a good idea to measure performance for different
 	// policies and chunk counts
 
+	ShardedPath.AppendSeparator();
 	ShardedPath.AppendAsciiRange(str, str + 3);
 
-	ShardedPath.Append(std::filesystem::path::preferred_separator);
+	ShardedPath.AppendSeparator();
 	ShardedPath.AppendAsciiRange(str + 3, str + 5);
 	Shard2len = ShardedPath.Size();
 
-	ShardedPath.Append(std::filesystem::path::preferred_separator);
+	ShardedPath.AppendSeparator();
 	ShardedPath.AppendAsciiRange(str + 5, str + 40);
 }
 
@@ -115,6 +117,9 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 	{
 		ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);
 
+		RwLock::ExclusiveLockScope _(LockForHash(ChunkHash));
+
+#if ZEN_PLATFORM_WINDOWS
 		const HANDLE ChunkFileHandle = FileRef.FileHandle;
 
 		auto DeletePayloadFileOnClose = [&] {
@@ -135,8 +140,6 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 		//
 		// Future improvement: maintain Bloom filter to avoid expensive file system probes?
 
-		RwLock::ExclusiveLockScope _(LockForHash(ChunkHash));
-
 		{
 			CAtlFile PayloadFile;
 
@@ -268,6 +271,49 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 				 ChunkHash);
 
 		DeletePayloadFileOnClose();
+#elif ZEN_PLATFORM_LINUX
+		std::filesystem::path SourcePath = PathFromHandle(FileRef.FileHandle);
+		std::filesystem::path DestPath	 = Name.ShardedPath.c_str();
+		int					  Ret		 = link(SourcePath.c_str(), DestPath.c_str());
+		if (Ret < 0 && zen::GetLastError() == ENOENT)
+		{
+			// Destination directory doesn't exist. Create it any try again.
+			CreateDirectories(DestPath.parent_path().c_str());
+			Ret = link(SourcePath.c_str(), DestPath.c_str());
+		}
+		int LinkError = zen::GetLastError();
+
+		// Unlink the file. If the path to unlink didn't exist someone else
+		// beat us to it and that is hunky-dory.
+		if (unlink(SourcePath.c_str()) < 0)
+		{
+			int UnlinkError = zen::GetLastError();
+			if (UnlinkError != ENOENT)
+			{
+				ZEN_WARN("unlink of CAS payload file failed ('{}')", GetSystemErrorAsString(UnlinkError));
+			}
+		}
+
+		// It is possible that someone beat us to it in linking the file. In that
+		// case a "file exists" error is okay. All others are not.
+		if (Ret < 0)
+		{
+			if (LinkError == EEXIST)
+			{
+				return CasStore::InsertResult{.New = false};
+			}
+
+			ZEN_WARN("link of CAS payload file failed ('{}'), falling back to regular write for insert of {}",
+					 GetSystemErrorAsString(LinkError),
+					 ChunkHash);
+		}
+		else
+		{
+			return CasStore::InsertResult{.New = true};
+		}
+#else
+#	error check link/unlink for this platform
+#endif	// ZEN_PLATFORM_*
 	}
 
 	return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash);
@@ -284,6 +330,7 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 	//
 	// Future improvement: maintain Bloom filter to avoid expensive file system probes?
 
+#if ZEN_PLATFORM_WINDOWS
 	CAtlFile PayloadFile;
 
 	HRESULT hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
@@ -298,9 +345,18 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 	}
 
 	PayloadFile.Close();
+#elif ZEN_PLATFORM_LINUX
+	if (access(Name.ShardedPath.c_str(), F_OK) == 0)
+	{
+		return CasStore::InsertResult{.New = false};
+	}
+#else
+#	error Check access() for this platform
+#endif
 
 	RwLock::ExclusiveLockScope _(LockForHash(ChunkHash));
 
+#if ZEN_PLATFORM_WINDOWS
 	// For now, use double-checked locking to see if someone else was first
 
 	hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
@@ -336,6 +392,49 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 	{
 		ThrowSystemException(hRes, "Failed to open shard file '{}'"_format(WideToUtf8(Name.ShardedPath)));
 	}
+#else
+	// Attempt to exclusively create the file.
+	auto InternalCreateFile = [&] { return open(Name.ShardedPath.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666); };
+	int Fd = InternalCreateFile();
+	if (Fd < 0)
+	{
+		switch (zen::GetLastError())
+		{
+			case EEXIST:
+				// Another thread has beat us to it so we're golden.
+				return {.New = false};
+
+			case ENOENT:
+				if (zen::CreateDirectories(std::string_view(Name.ShardedPath.c_str(), Name.Shard2len)))
+				{
+					Fd = InternalCreateFile();
+					if (Fd >= 0)
+					{
+						break;
+					}
+				}
+				ThrowLastError("Failed creating shard directory '{}'"_format(Name.ShardedPath));
+
+			default:
+				ThrowLastError("Unexpected error occurred opening shard file '{}'"_format(Name.ShardedPath));
+		}
+	}
+
+	struct FdWrapper
+	{
+		~FdWrapper() { Close(); }
+		void Write(const void* Cursor, size_t Size) { (void)!write(Fd, Cursor, Size); }
+		void Close()
+		{
+			if (Fd >= 0)
+			{
+				close(Fd);
+				Fd = -1;
+			}
+		}
+		int Fd;
+	} PayloadFile = {Fd};
+#endif	// ZEN_PLATFORM_WINDOWS
 
 	size_t ChunkRemain = ChunkSize;
 	auto   ChunkCursor = reinterpret_cast<const uint8_t*>(ChunkData);
@@ -437,13 +536,13 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile&
 	struct Visitor : public FileSystemTraversal::TreeVisitor
 	{
 		Visitor(const std::filesystem::path& RootDir) : RootDirectory(RootDir) {}
-		virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
 		{
 			ZEN_UNUSED(FileSize);
 
 			std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory);
 
-			std::wstring PathString = RelPath.native();
+			std::filesystem::path::string_type PathString = RelPath.native();
 
 			if ((PathString.size() == (3 + 2 + 1)) && (File.size() == (40 - 3 - 2)))
 			{
@@ -453,12 +552,15 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile&
 				}
 				PathString.append(File);
 
-				StringBuilder<64> Utf8;
-				WideToUtf8(PathString, Utf8);
-
 				// TODO: should validate that we're actually dealing with a valid hex string here
 
+#if ZEN_PLATFORM_WINDOWS
+				StringBuilder<64> Utf8;
+				WideToUtf8(PathString, Utf8);
 				IoHash NameHash = IoHash::FromHexString({Utf8.Data(), Utf8.Size()});
+#else
+				IoHash NameHash = IoHash::FromHexString(PathString);
+#endif
 
 				BasicFile		PayloadFile;
 				std::error_code Ec;
@@ -471,8 +573,7 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile&
 			}
 		}
 
-		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
-									[[maybe_unused]] const std::wstring_view&	  DirectoryName)
+		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, [[maybe_unused]] const path_view& DirectoryName)
 		{
 			return true;
 		}