move all storage-related services into storage tree (#571)

* move all storage-related services into storage tree
* move config into config/
* also move admin service into storage since it mostly has storage related functionality
* header consolidation

1 files changed, 203 insertions, 0 deletions

diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h
new file mode 100644
index 000000000..ca0cf4135
--- /dev/null
+++ b/src/zenserver/storage/storageconfig.h
@@ -0,0 +1,203 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "config/config.h"
+
+namespace zen {
+
+struct ZenUpstreamJupiterConfig
+{
+	std::string Name;
+	std::string Url;
+	std::string OAuthUrl;
+	std::string OAuthClientId;
+	std::string OAuthClientSecret;
+	std::string OpenIdProvider;
+	std::string AccessToken;
+	std::string Namespace;
+	std::string DdcNamespace;
+};
+
+struct ZenUpstreamZenConfig
+{
+	std::string				 Name;
+	std::vector<std::string> Urls;
+	std::vector<std::string> Dns;
+};
+
+enum class UpstreamCachePolicy : uint8_t
+{
+	Disabled  = 0,
+	Read	  = 1 << 0,
+	Write	  = 1 << 1,
+	ReadWrite = Read | Write
+};
+
+struct ZenUpstreamCacheConfig
+{
+	ZenUpstreamJupiterConfig JupiterConfig;
+	ZenUpstreamZenConfig	 ZenConfig;
+	int32_t					 UpstreamThreadCount		= 4;
+	int32_t					 ConnectTimeoutMilliseconds = 5000;
+	int32_t					 TimeoutMilliseconds		= 0;
+	UpstreamCachePolicy		 CachePolicy				= UpstreamCachePolicy::ReadWrite;
+};
+
+struct ZenCacheEvictionPolicy
+{
+	int32_t MaxDurationSeconds = 24 * 60 * 60;
+};
+
+struct ZenProjectStoreEvictionPolicy
+{
+	int32_t MaxDurationSeconds = 7 * 24 * 60 * 60;
+};
+
+struct ZenBuildStoreEvictionPolicy
+{
+	int32_t MaxDurationSeconds = 3 * 24 * 60 * 60;
+};
+
+struct ZenGcConfig
+{
+	//	ZenCasEvictionPolicy   Cas;
+	ZenCacheEvictionPolicy		  Cache;
+	ZenProjectStoreEvictionPolicy ProjectStore;
+	ZenBuildStoreEvictionPolicy	  BuildStore;
+	int32_t						  MonitorIntervalSeconds			= 30;
+	int32_t						  IntervalSeconds					= 0;
+	bool						  CollectSmallObjects				= true;
+	bool						  Enabled							= true;
+	uint64_t					  DiskReserveSize					= 1ul << 28;
+	uint64_t					  DiskSizeSoftLimit					= 0;
+	int32_t						  LightweightIntervalSeconds		= 0;
+	uint64_t					  MinimumFreeDiskSpaceToAllowWrites = 1ul << 28;
+	bool						  UseGCV2							= false;
+	uint32_t					  CompactBlockUsageThresholdPercent = 90;
+	bool						  Verbose							= false;
+	bool						  SingleThreaded					= false;
+	static constexpr uint16_t	  GcMaxAttachmentPassCount			= 256;
+	uint16_t					  AttachmentPassCount				= 1;
+	bool						  StoreCacheAttachmentMetaData		= false;
+	bool						  StoreProjectAttachmentMetaData	= false;
+	bool						  EnableValidation					= true;
+};
+
+struct ZenOpenIdProviderConfig
+{
+	std::string Name;
+	std::string Url;
+	std::string ClientId;
+};
+
+struct ZenAuthConfig
+{
+	std::vector<ZenOpenIdProviderConfig> OpenIdProviders;
+};
+
+struct ZenObjectStoreConfig
+{
+	struct BucketConfig
+	{
+		std::string			  Name;
+		std::filesystem::path Directory;
+	};
+
+	std::vector<BucketConfig> Buckets;
+};
+
+struct ZenStructuredCacheBucketConfig
+{
+	uint64_t MaxBlockSize		   = 1ull << 30;
+	uint32_t PayloadAlignment	   = 1u << 4;
+	uint64_t MemCacheSizeThreshold = 1 * 1024;
+	uint64_t LargeObjectThreshold  = 128 * 1024;
+	bool	 LimitOverwrites	   = false;
+};
+
+struct ZenStructuredCacheConfig
+{
+	bool																Enabled			 = true;
+	bool																WriteLogEnabled	 = false;
+	bool																AccessLogEnabled = false;
+	std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>> PerBucketConfigs;
+	ZenStructuredCacheBucketConfig										BucketConfig;
+	uint64_t															MemTargetFootprintBytes = 512 * 1024 * 1024;
+	uint64_t															MemTrimIntervalSeconds	= 60;
+	uint64_t MemMaxAgeSeconds = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count());
+};
+
+struct ZenProjectStoreConfig
+{
+	bool StoreCacheAttachmentMetaData	= false;
+	bool StoreProjectAttachmentMetaData = false;
+};
+
+struct ZenBuildStoreConfig
+{
+	bool	 Enabled		   = false;
+	uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u;  // 1TB
+};
+
+struct ZenWorkspacesConfig
+{
+	bool Enabled				   = false;
+	bool AllowConfigurationChanges = false;
+};
+
+struct ZenStorageServerOptions : public ZenServerOptions
+{
+	ZenUpstreamCacheConfig	 UpstreamCacheConfig;
+	ZenGcConfig				 GcConfig;
+	ZenAuthConfig			 AuthConfig;
+	ZenObjectStoreConfig	 ObjectStoreConfig;
+	ZenStructuredCacheConfig StructuredCacheConfig;
+	ZenProjectStoreConfig	 ProjectStoreConfig;
+	ZenBuildStoreConfig		 BuildStoreConfig;
+	ZenWorkspacesConfig		 WorksSpacesConfig;
+	std::filesystem::path	 PluginsConfigFile;	 // Path to plugins config file
+	std::filesystem::path	 BaseSnapshotDir;	 // Path to server state snapshot (will be copied into data dir on start)
+	bool					 ObjectStoreEnabled = false;
+	std::string				 ScrubOptions;
+};
+
+void ParseConfigFile(const std::filesystem::path& Path,
+					 ZenStorageServerOptions&	  ServerOptions,
+					 const cxxopts::ParseResult&  CmdLineResult,
+					 std::string_view			  OutputConfigFile);
+
+void ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort);
+void ValidateOptions(ZenStorageServerOptions& ServerOptions);
+
+struct ZenStorageServerCmdLineOptions
+{
+	// Note to those adding future options; std::filesystem::path-type options
+	// must be read into a std::string first. As of cxxopts-3.0.0 it uses a >>
+	// stream operator to convert argv value into the options type. std::fs::path
+	// expects paths in streams to be quoted but argv paths are unquoted. By
+	// going into a std::string first, paths with whitespace parse correctly.
+	std::string PluginsConfigFile;
+	std::string BaseSnapshotDir;
+
+	void AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions);
+	void ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions);
+
+	std::string OpenIdProviderName;
+	std::string OpenIdProviderUrl;
+	std::string OpenIdClientId;
+
+	void AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions);
+
+	std::string UpstreamCachePolicyOptions;
+
+	void AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions);
+
+	void AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions);
+
+	std::vector<std::string> BucketConfigs;
+
+	void AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions);
+	void AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions);
+	void AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions);
+};
+
+}  // namespace zen