Merge branch 'gc' of https://github.com/EpicGames/zen into gc

13 files changed, 173 insertions, 38 deletions

diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index b8e8eff55..a642e2cf6 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -840,7 +840,7 @@ TEST_CASE("filesystem")
 
 	// GetExePath -- this is not a great test as it's so dependent on where the this code gets linked in
 	path	   BinPath	   = GetRunningExecutablePath();
-	const bool ExpectedExe = BinPath.stem() == "zencore-test" || BinPath.stem() == "zenserver-test" || BinPath.stem() == "zenstore-test";
+	const bool ExpectedExe = ToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver";
 	CHECK(ExpectedExe);
 	CHECK(is_regular_file(BinPath));
 
diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h
index 640c3fe74..1f87dc639 100644
--- a/zencore/include/zencore/sharedbuffer.h
+++ b/zencore/include/zencore/sharedbuffer.h
@@ -143,6 +143,12 @@ public:
 
 	/** Make a non-owned view of the input */
 	[[nodiscard]] inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); }
+	/** Make a non-owning view of the memory of the contiguous container. */
+	[[nodiscard]] inline static SharedBuffer MakeView(const std::ranges::contiguous_range auto& Container)
+	{
+		std::span Span = Container;
+		return MakeView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type));
+	}
 	/** Make a non-owned view of the input */
 	[[nodiscard]] ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size);
 	/** Make a non-owned view of the input */
diff --git a/zencore/memory.cpp b/zencore/memory.cpp
index 62c81076d..14ea7ca1d 100644
--- a/zencore/memory.cpp
+++ b/zencore/memory.cpp
@@ -185,13 +185,13 @@ TEST_CASE("ChunkingLinearAllocator")
 TEST_CASE("MemoryView")
 {
 	{
-		uint8_t	   Array1[16];
+		uint8_t	   Array1[16] = {};
 		MemoryView View1 = MakeMemoryView(Array1);
 		CHECK(View1.GetSize() == 16);
 	}
 
 	{
-		uint32_t   Array2[16];
+		uint32_t   Array2[16] = {};
 		MemoryView View2 = MakeMemoryView(Array2);
 		CHECK(View2.GetSize() == 64);
 	}
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 7b65cdc78..1b845f0f4 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -68,6 +68,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #endif
 
 using namespace fmt::literals;
+using namespace std::literals;
 
 /*
 
@@ -128,10 +129,10 @@ public:
 		m_Headers.reserve(16);
 
 		zen::ExtendableStringBuilder<256> RequestBody;
-		RequestBody << "GET " << Path << " HTTP/1.1\r\n";
-		RequestBody << "Host: " << Server << "\r\n";
-		RequestBody << "Accept: */*\r\n";
-		RequestBody << "Connection: " << (m_KeepAlive ? "keep-alive" : "close") << "\r\n\r\n";	// TODO: support keep-alive
+		RequestBody << "GET "sv << Path << " HTTP/1.1\r\n"sv;
+		RequestBody << "Host: "sv << Server << "\r\n"sv;
+		RequestBody << "Accept: */*\r\n"sv;
+		RequestBody << "Connection: "sv << (m_KeepAlive ? "keep-alive"sv : "close"sv) << "\r\n\r\n"sv;	// TODO: support keep-alive
 
 		m_RequestBody = RequestBody;
 
@@ -462,12 +463,12 @@ using namespace spdlog;
 using namespace spdlog::details;
 using namespace std::literals;
 
-class full_formatter final : public spdlog::formatter
+class full_test_formatter final : public spdlog::formatter
 {
 public:
-	full_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch) : m_Epoch(Epoch), m_LogId(LogId) {}
+	full_test_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch) : m_Epoch(Epoch), m_LogId(LogId) {}
 
-	virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_formatter>(m_LogId, m_Epoch); }
+	virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_test_formatter>(m_LogId, m_Epoch); }
 
 	static constexpr bool UseDate = false;
 
@@ -682,7 +683,7 @@ main(int argc, char** argv)
 	zen::logging::InitializeLogging();
 
 	spdlog::set_level(spdlog::level::debug);
-	spdlog::set_formatter(std::make_unique<::logging::full_formatter>("test", std::chrono::system_clock::now()));
+	spdlog::set_formatter(std::make_unique<::logging::full_test_formatter>("test", std::chrono::system_clock::now()));
 
 	std::filesystem::path ProgramBaseDir = std::filesystem::path(argv[0]).parent_path();
 	std::filesystem::path TestBaseDir	 = ProgramBaseDir.parent_path().parent_path() / ".test";
diff --git a/zenserver/cache/cacheagent.cpp b/zenserver/cache/cacheagent.cpp
deleted file mode 100644
index f4d1cabe6..000000000
--- a/zenserver/cache/cacheagent.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "cacheagent.h"
-
-#include <gsl/gsl-lite.hpp>
diff --git a/zenserver/cache/cacheagent.h b/zenserver/cache/cacheagent.h
deleted file mode 100644
index 145d0f79f..000000000
--- a/zenserver/cache/cacheagent.h
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-class CacheAgent
-{
-public:
-private:
-};
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index f964c3102..b120f3955 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -6,21 +6,28 @@
 #include <zencore/windows.h>
 
 #include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinarypackage.h>
 #include <zencore/compactbinaryvalidation.h>
+#include <zencore/compress.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
 #include <zencore/thread.h>
 #include <zenstore/basicfile.h>
 #include <zenstore/cas.h>
 #include <zenstore/caslog.h>
+#include <zenstore/cidstore.h>
 #include <zenstore/gc.h>
 
 #include <concepts>
 #include <filesystem>
 #include <memory_resource>
+#include <ranges>
 #include <unordered_map>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -236,7 +243,12 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
 void
 ZenCacheMemoryLayer::GatherReferences(GcContext& GcCtx)
 {
-	ZEN_UNUSED(GcCtx);
+	RwLock::SharedLockScope _(m_Lock);
+
+	for (auto& Kv : m_Buckets)
+	{
+		Kv.second.GatherReferences(GcCtx);
+	}
 }
 
 void
@@ -1079,4 +1091,58 @@ ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx)
 	}
 }
 
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("z$.store")
+{
+	using namespace fmt::literals;
+	using namespace std::literals;
+
+	ScopedTemporaryDirectory TempDir;
+
+	CasGc Gc;
+
+	ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
+
+	const int kIterationCount = 100;
+
+	for (int i = 0; i < kIterationCount; ++i)
+	{
+		const IoHash Key = IoHash::HashBuffer(&i, sizeof i);
+
+		CbObjectWriter Cbo;
+		Cbo << "hey" << i;
+		CbObject Obj = Cbo.Save();
+
+		ZenCacheValue Value;
+		Value.Value = Obj.GetBuffer().AsIoBuffer();
+		Value.Value.SetContentType(ZenContentType::kCbObject);
+
+		Zcs.Put("test_bucket"sv, Key, Value);
+	}
+
+	for (int i = 0; i < kIterationCount; ++i)
+	{
+		const IoHash Key = IoHash::HashBuffer(&i, sizeof i);
+
+		ZenCacheValue Value;
+		Zcs.Get("test_bucket"sv, Key, /* out */ Value);
+
+		REQUIRE(Value.Value);
+		CHECK(Value.Value.GetContentType() == ZenContentType::kCbObject);
+		CHECK_EQ(ValidateCompactBinary(Value.Value, CbValidateMode::All), CbValidateError::None);
+		CbObject Obj = LoadCompactBinaryObject(Value.Value);
+		CHECK_EQ(Obj["hey"].AsInt32(), i);
+	}
+}
+
+#endif
+
+void
+z$_forcelink()
+{
+}
+
 }  // namespace zen
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index da3e74126..6beecf78b 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -105,7 +105,7 @@ private:
 class ZenCacheDiskLayer
 {
 public:
-	ZenCacheDiskLayer(const std::filesystem::path& RootDir);
+	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir);
 	~ZenCacheDiskLayer();
 
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -149,4 +149,6 @@ private:
 	uint64_t			  m_LastScrubTime		   = 0;
 };
 
+void z$_forcelink();
+
 }  // namespace zen
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index cbdbebb03..f512f8015 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -234,6 +234,13 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
 
 	options.add_option("cache",
 					   "",
+					   "upstream-zen-dns",
+					   "DNS that resolves to one or more Zen server instance(s)",
+					   cxxopts::value<std::vector<std::string>>(ServiceConfig.UpstreamCacheConfig.ZenConfig.Dns)->default_value(""),
+					   "");
+
+	options.add_option("cache",
+					   "",
 					   "upstream-thread-count",
 					   "Number of threads used for upstream procsssing",
 					   cxxopts::value<int>(ServiceConfig.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"),
diff --git a/zenserver/config.h b/zenserver/config.h
index 2a5a17fb9..72a4f31bb 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -42,6 +42,7 @@ struct ZenUpstreamJupiterConfig
 struct ZenUpstreamZenConfig
 {
 	std::vector<std::string> Urls;
+	std::vector<std::string> Dns;
 };
 
 enum class UpstreamCachePolicy : uint8_t
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index e98ddb0fd..8da781a59 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -120,6 +120,37 @@ namespace zen {
 using namespace std::literals;
 using namespace fmt::literals;
 
+namespace utils {
+	asio::error_code ResolveHostname(asio::io_context&		   Ctx,
+									 std::string_view		   Host,
+									 std::string_view		   DefaultPort,
+									 std::vector<std::string>& OutEndpoints)
+	{
+		std::string_view Port = DefaultPort;
+
+		if (const size_t Idx = Host.find(":"); Idx != std::string_view::npos)
+		{
+			Port = Host.substr(Idx + 1);
+			Host = Host.substr(0, Idx);
+		}
+
+		asio::ip::tcp::resolver Resolver(Ctx);
+
+		asio::error_code					  ErrorCode;
+		asio::ip::tcp::resolver::results_type Endpoints = Resolver.resolve(Host, Port, ErrorCode);
+
+		if (!ErrorCode)
+		{
+			for (const asio::ip::tcp::endpoint& Ep : Endpoints)
+			{
+				OutEndpoints.push_back("http://{}:{}"_format(Ep.address().to_string(), Ep.port()));
+			}
+		}
+
+		return ErrorCode;
+	}
+}  // namespace utils
+
 class ZenServer : public IHttpStatusProvider
 {
 public:
@@ -642,12 +673,29 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
 
 		UpstreamCache = zen::MakeUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore);
 
-		if (!UpstreamConfig.ZenConfig.Urls.empty())
+		// Zen upstream
 		{
-			std::unique_ptr<zen::UpstreamEndpoint> ZenEndpoint = zen::MakeZenUpstreamEndpoint(UpstreamConfig.ZenConfig.Urls);
-			UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint));
+			std::vector<std::string> ZenUrls = UpstreamConfig.ZenConfig.Urls;
+			if (!UpstreamConfig.ZenConfig.Dns.empty())
+			{
+				for (const std::string& Dns : UpstreamConfig.ZenConfig.Dns)
+				{
+					const asio::error_code Err = zen::utils::ResolveHostname(m_IoContext, Dns, "1337"sv, ZenUrls);
+					if (Err)
+					{
+						ZEN_ERROR("resolve '{}' FAILED, reason '{}'", Err.message());
+					}
+				}
+			}
+
+			if (!ZenUrls.empty())
+			{
+				std::unique_ptr<zen::UpstreamEndpoint> ZenEndpoint = zen::MakeZenUpstreamEndpoint(ZenUrls);
+				UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint));
+			}
 		}
 
+		// Jupiter upstream
 		{
 			zen::CloudCacheClientOptions Options;
 			if (UpstreamConfig.JupiterConfig.UseProductionSettings)
@@ -864,6 +912,22 @@ ZenWindowsService::Run()
 	return 0;
 }
 
+#if ZEN_WITH_TESTS
+int
+test_main(int argc, char** argv)
+{
+	zen::zencore_forcelinktests();
+	zen::zenhttp_forcelinktests();
+	zen::zenstore_forcelinktests();
+
+	zen::logging::InitializeLogging();
+
+	spdlog::set_level(spdlog::level::debug);
+
+	return doctest::Context(argc, argv).run();
+}
+#endif
+
 int
 main(int argc, char* argv[])
 {
@@ -873,6 +937,16 @@ main(int argc, char* argv[])
 	mi_version();
 #endif
 
+#if ZEN_WITH_TESTS
+	if (argc >= 2)
+	{
+		if (argv[1] == "test"sv)
+		{
+			return test_main(argc, argv);
+		}
+	}
+#endif
+
 	try
 	{
 		ZenServerOptions GlobalOptions;
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index b670582e7..d954d3f8d 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -120,7 +120,6 @@
     <ClInclude Include="testing\httptest.h" />
     <ClInclude Include="upstream\jupiter.h" />
     <ClInclude Include="projectstore.h" />
-    <ClInclude Include="cache\cacheagent.h" />
     <ClInclude Include="testing\launch.h" />
     <ClInclude Include="casstore.h" />
     <ClInclude Include="diag\diagsvcs.h" />
@@ -142,7 +141,6 @@
     <ClCompile Include="monitoring\httpstats.cpp" />
     <ClCompile Include="monitoring\httpstatus.cpp" />
     <ClCompile Include="projectstore.cpp" />
-    <ClCompile Include="cache\cacheagent.cpp" />
     <ClCompile Include="sos\sos.cpp" />
     <ClCompile Include="testing\httptest.cpp" />
     <ClCompile Include="upstream\jupiter.cpp" />
diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters
index b87fa0016..04c6267ba 100644
--- a/zenserver/zenserver.vcxproj.filters
+++ b/zenserver/zenserver.vcxproj.filters
@@ -5,9 +5,6 @@
     <ClInclude Include="projectstore.h" />
     <ClInclude Include="casstore.h" />
     <ClInclude Include="testing\launch.h" />
-    <ClInclude Include="cache\cacheagent.h">
-      <Filter>cache</Filter>
-    </ClInclude>
     <ClInclude Include="diag\diagsvcs.h">
       <Filter>diag</Filter>
     </ClInclude>
@@ -49,9 +46,6 @@
     <ClCompile Include="zenserver.cpp" />
     <ClCompile Include="projectstore.cpp" />
     <ClCompile Include="casstore.cpp" />
-    <ClCompile Include="cache\cacheagent.cpp">
-      <Filter>cache</Filter>
-    </ClCompile>
     <ClCompile Include="experimental\usnjournal.cpp">
       <Filter>experimental</Filter>
     </ClCompile>