path: root/zenserver/projectstore.h

// Copyright Epic Games, Inc. All Rights Reserved.

#pragma once

#include <zencore/logging.h>
#include <zencore/uid.h>
#include <zencore/xxhash.h>
#include <zenhttp/httpserver.h>
#include <zenstore/cidstore.h>
#include <zenstore/gc.h>
#include <zenutil/cache/cachekey.h>

#include <filesystem>
#include <map>
#include <optional>
#include <string>

ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
ZEN_THIRD_PARTY_INCLUDES_END

namespace zen {

class CbPackage;
class ZenCacheStore;

struct OplogEntry
{
	uint32_t OpLsn;
	uint32_t OpCoreOffset;	// note: Multiple of alignment!
	uint32_t OpCoreSize;
	uint32_t OpCoreHash;  // Used as checksum
	XXH3_128 OpKeyHash;	  // XXH128_canonical_t

	inline Oid OpKeyAsOId() const
	{
		Oid Id;
		memcpy(Id.OidBits, &OpKeyHash, sizeof Id.OidBits);
		return Id;
	}
};

struct OplogEntryAddress
{
	uint64_t Offset;
	uint64_t Size;
};

static_assert(IsPow2(sizeof(OplogEntry)));

/** Project Store

	A project store consists of a number of Projects.

	Each project contains a number of oplogs (short for "operation log"). UE uses
	one oplog per target platform to store the output of the cook process.

	An oplog consists of a sequence of "op" entries. Each entry is a structured object
	containing references to attachments. Attachments are typically the serialized
	package data split into separate chunks for bulk data, exports and header
	information.
 */
class ProjectStore : public RefCounted, public GcStorage, public GcContributor
{
	struct OplogStorage;

public:
	ProjectStore(CidStore& Store, ZenCacheStore& CacheStore, std::filesystem::path BasePath, GcManager& Gc);
	~ProjectStore();

	struct Project;

	struct Oplog
	{
		Oplog(std::string_view Id, Project* Project, CidStore& CidStore, ZenCacheStore& CacheStore, std::filesystem::path BasePath);
		~Oplog();

		[[nodiscard]] static bool ExistsAt(std::filesystem::path BasePath);

		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
		void IterateOplog(std::function<void(CbObject)>&& Fn);
		std::optional<CbObject> GetOpByKey(const Oid& Key);
		std::optional<CbObject> GetOpByIndex(int Index);

		IoBuffer FindChunk(Oid ChunkId);

		inline static const uint32_t kInvalidOp = ~0u;

		/** Persist a new oplog entry
		 *
		 * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected
		 */
		uint32_t AppendNewOplogEntry(CbPackage Op);

		uint32_t AppendNewOplogEntry(CbObject Core);

		enum UpdateType
		{
			kUpdateNewEntry,
			kUpdateReplay
		};

		/** Update tracking metadata for a new oplog entry
		 *
		 * This is used during replay (and gets called as part of new op append)
		 *
		 * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected
		 */
		uint32_t RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry, UpdateType TypeOfUpdate);

		/** Scan oplog and register each entry, thus updating the in-memory tracking tables
		 */
		void ReplayLog();

		const std::string& OplogId() const { return m_OplogId; }

		const std::filesystem::path& TempPath() const { return m_TempPath; }

		spdlog::logger& Log() { return m_OuterProject->Log(); }
		void			Flush();
		void			Scrub(ScrubContext& Ctx) const;
		void			GatherReferences(GcContext& GcCtx);

		std::size_t OplogCount() const
		{
			RwLock::SharedLockScope _(m_OplogLock);
			return m_LatestOpMap.size();
		}

		std::filesystem::path PrepareForDelete(bool MoveFolder);

	private:
		struct FileMapEntry
		{
			std::string ServerPath;
			std::string ClientPath;
		};

		struct CacheMapEntry
		{
			std::string Namespace;
			CacheKey	CacheKey;
			Oid			ValueId;
			IoHash		Cid;
		};

		template<class V>
		using OidMap = tsl::robin_map<Oid, V, Oid::Hasher>;

		Project*			  m_OuterProject = nullptr;
		CidStore&			  m_CidStore;
		ZenCacheStore&		  m_CacheStore;
		std::filesystem::path m_BasePath;
		std::filesystem::path m_TempPath;

		mutable RwLock					 m_OplogLock;
		OidMap<IoHash>					 m_ChunkMap;		 // output data chunk id -> CAS address
		OidMap<IoHash>					 m_MetaMap;			 // meta chunk id -> CAS address
		OidMap<FileMapEntry>			 m_FileMap;			 // file id -> file map entry
		OidMap<CacheMapEntry>			 m_CacheMap;		 // chunk id -> CAS address
		int32_t							 m_ManifestVersion;	 // File system manifest version
		std::map<int, OplogEntryAddress> m_OpAddressMap;	 // Index LSN -> op data in ops blob file
		OidMap<int>						 m_LatestOpMap;		 // op key -> latest op LSN for key

		RefPtr<OplogStorage> m_Storage;
		std::string			 m_OplogId;

		bool AddFileMapping(const RwLock::ExclusiveLockScope& OplogLock,
							Oid								  FileId,
							IoHash							  Hash,
							std::string_view				  ServerPath,
							std::string_view				  ClientPath);
		void AddChunkMapping(const RwLock::ExclusiveLockScope& OplogLock, Oid ChunkId, IoHash Hash);
		void AddMetaMapping(const RwLock::ExclusiveLockScope& OplogLock, Oid ChunkId, IoHash Hash);
		bool AddCacheMapping(const RwLock::ExclusiveLockScope& OplogLock,
							 std::string_view				   Namespace,
							 const CacheKey&				   CacheKey,
							 const Oid&						   ValueId,
							 const Oid&						   ChunkId);

		IoHash ResolveCacheMapping(std::string_view Namespace, const CacheKey& CacheKey, const Oid& ValueId);
	};

	struct Project : public RefCounted
	{
		std::string			  Identifier;
		std::filesystem::path RootDir;
		std::string			  EngineRootDir;
		std::string			  ProjectRootDir;
		std::string			  ProjectFilePath;

		Oplog*					 NewOplog(std::string_view OplogId);
		Oplog*					 OpenOplog(std::string_view OplogId);
		void					 DeleteOplog(std::string_view OplogId);
		void					 IterateOplogs(std::function<void(const Oplog&)>&& Fn) const;
		void					 IterateOplogs(std::function<void(Oplog&)>&& Fn);
		std::vector<std::string> ScanForOplogs() const;
		bool					 IsExpired() const;

		Project(ProjectStore* PrjStore, CidStore& CidStore, ZenCacheStore& CacheStore, std::filesystem::path BasePath);
		virtual ~Project();

		void					  Read();
		void					  Write();
		[[nodiscard]] static bool Exists(std::filesystem::path BasePath);
		void					  Flush();
		void					  Scrub(ScrubContext& Ctx);
		spdlog::logger&			  Log();
		void					  GatherReferences(GcContext& GcCtx);
		bool					  PrepareForDelete(std::filesystem::path& OutDeletePath);

	private:
		ProjectStore*								  m_ProjectStore;
		CidStore&									  m_CidStore;
		ZenCacheStore&								  m_CacheStore;
		mutable RwLock								  m_ProjectLock;
		std::map<std::string, std::unique_ptr<Oplog>> m_Oplogs;
		std::vector<std::unique_ptr<Oplog>>			  m_DeletedOplogs;
		std::filesystem::path						  m_OplogStoragePath;

		std::filesystem::path BasePathForOplog(std::string_view OplogId);
	};

	//	Oplog* OpenProjectOplog(std::string_view ProjectId, std::string_view OplogId);

	Ref<Project> OpenProject(std::string_view ProjectId);
	Ref<Project> NewProject(std::filesystem::path BasePath,
							std::string_view	  ProjectId,
							std::string_view	  RootDir,
							std::string_view	  EngineRootDir,
							std::string_view	  ProjectRootDir,
							std::string_view	  ProjectFilePath);
	bool		 DeleteProject(std::string_view ProjectId);
	bool		 Exists(std::string_view ProjectId);
	void		 Flush();
	void		 Scrub(ScrubContext& Ctx);
	void		 DiscoverProjects();
	void		 IterateProjects(std::function<void(Project& Prj)>&& Fn);

	spdlog::logger&				 Log() { return m_Log; }
	const std::filesystem::path& BasePath() const { return m_ProjectBasePath; }

	virtual void		  GatherReferences(GcContext& GcCtx) override;
	virtual void		  CollectGarbage(GcContext& GcCtx) override;
	virtual GcStorageSize StorageSize() const override;

	CbArray			 GetProjectsList();
	HttpResponseCode GetProjectFiles(const std::string_view ProjectId,
									 const std::string_view OplogId,
									 bool					FilterClient,
									 CbObject&				OutPayload);
	HttpResponseCode GetChunkInfo(const std::string_view ProjectId,
								  const std::string_view OplogId,
								  const std::string_view ChunkId,
								  CbObject&				 OutPayload);
	HttpResponseCode GetChunk(const std::string_view ProjectId,
							  const std::string_view OplogId,
							  const std::string_view ChunkId,
							  uint64_t				 Offset,
							  uint64_t				 Size,
							  ZenContentType		 AcceptType,
							  IoBuffer&				 OutChunk);
	HttpResponseCode GetChunk(const std::string_view Cid, ZenContentType AcceptType, IoBuffer& OutChunk);

private:
	spdlog::logger&						m_Log;
	CidStore&							m_CidStore;
	ZenCacheStore&						m_CacheStore;
	std::filesystem::path				m_ProjectBasePath;
	RwLock								m_ProjectsLock;
	std::map<std::string, Ref<Project>> m_Projects;

	std::filesystem::path BasePathForProject(std::string_view ProjectId);
};

//////////////////////////////////////////////////////////////////////////
//
//  {project}	a project identifier
//  {target}	a variation of the project, typically a build target
//  {lsn}		oplog entry sequence number
//
//  /prj/{project}
//  /prj/{project}/oplog/{target}
//  /prj/{project}/oplog/{target}/{lsn}
//
// oplog entry
//
// id: {id}
// key: {}
// meta: {}
// data: []
// refs:
//

class HttpProjectService : public HttpService
{
public:
	HttpProjectService(CidStore& Store, ProjectStore* InProjectStore);
	~HttpProjectService();

	virtual const char* BaseUri() const override;
	virtual void		HandleRequest(HttpServerRequest& Request) override;

private:
	inline spdlog::logger& Log() { return m_Log; }

	spdlog::logger&	  m_Log;
	CidStore&		  m_CidStore;
	HttpRequestRouter m_Router;
	Ref<ProjectStore> m_ProjectStore;
};

void prj_forcelink();

}  // namespace zen