Cache request record/replay (#198)

This adds recording and playback of cache request with full data - both get and put operations can be replayed.

Invoke via web request.

`<host>/z$/exec$/start-recording?<disk-storage-path>`
`<host>/z$/exec$/stop-recording`
`<host>/z$/exec$/replay-recording?<thread-count>&<disk-storage-path>`

2 files changed, 29 insertions, 6 deletions

diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index 5e376ffda..0aa478404 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -947,7 +947,28 @@ PathFromHandle(void* NativeHandle)
 		return std::filesystem::path();
 	}
 
-	const DWORD RequiredLengthIncludingNul = GetFinalPathNameByHandleW(NativeHandle, nullptr, 0, FILE_NAME_OPENED);
+	auto GetFinalPathNameByHandleWRetry = [](HANDLE hFile, LPWSTR lpszFilePath, DWORD cchFilePath, DWORD dwFlags) -> DWORD {
+		while (true)
+		{
+			DWORD Res = GetFinalPathNameByHandleW(hFile, lpszFilePath, cchFilePath, dwFlags);
+			if (Res == 0)
+			{
+				DWORD LastError = zen::GetLastError();
+				// Under heavy concurrent loads we might get access denied on a file handle while trying to get path name.
+				// Retry if that is the case.
+				if (LastError != ERROR_ACCESS_DENIED)
+				{
+					ThrowSystemError(LastError, fmt::format("failed to get path from file handle {}", hFile));
+				}
+				// Retry
+				continue;
+			}
+			ZEN_ASSERT(Res != 1);  // We don't accept empty path names
+			return Res;
+		}
+	};
+
+	DWORD RequiredLengthIncludingNul = GetFinalPathNameByHandleWRetry(NativeHandle, nullptr, 0, FILE_NAME_OPENED);
 	if (RequiredLengthIncludingNul == 0)
 	{
 		ThrowLastError(fmt::format("failed to get path from file handle {}", NativeHandle));
@@ -956,8 +977,7 @@ PathFromHandle(void* NativeHandle)
 	std::wstring FullPath;
 	FullPath.resize(RequiredLengthIncludingNul - 1);
 
-	const DWORD FinalLength = GetFinalPathNameByHandleW(NativeHandle, FullPath.data(), RequiredLengthIncludingNul, FILE_NAME_OPENED);
-
+	const DWORD FinalLength = GetFinalPathNameByHandleWRetry(NativeHandle, FullPath.data(), RequiredLengthIncludingNul, FILE_NAME_OPENED);
 	ZEN_UNUSED(FinalLength);
 
 	return FullPath;
diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h
index 3c1821a62..2aad22061 100644
--- a/zencore/include/zencore/thread.h
+++ b/zencore/include/zencore/thread.h
@@ -161,13 +161,16 @@ public:
 		}
 	}
 
-	void Wait()
+	std::ptrdiff_t Remaining() const { return Counter.load(); }
+
+	bool Wait(int TimeoutMs = -1)
 	{
 		std::ptrdiff_t Old = Counter.load();
-		if (Old != 0)
+		if (Old == 0)
 		{
-			Complete.Wait();
+			return true;
 		}
+		return Complete.Wait(TimeoutMs);
 	}
 
 private: