path: root/zenserver/projectstore.h

// Copyright Epic Games, Inc. All Rights Reserved.

#pragma once

#include <zencore/logging.h>
#include <zencore/uid.h>
#include <zencore/xxhash.h>
#include <zenhttp/httpserver.h>
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
#include <zenstore/cidstore.h>
#include <zenstore/gc.h>

#include <filesystem>
#include <map>
#include <optional>
#include <string>

ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
ZEN_THIRD_PARTY_INCLUDES_END

namespace zen {

class CbPackage;

struct OplogEntry
{
	uint32_t OpLsn;
	uint32_t OpCoreOffset;	// note: Multiple of alignment!
	uint32_t OpCoreSize;
	uint32_t OpCoreHash;  // Used as checksum
	XXH3_128 OpKeyHash;	  // XXH128_canonical_t

	inline Oid OpKeyAsOId() const
	{
		Oid Id;
		memcpy(Id.OidBits, &OpKeyHash, sizeof Id.OidBits);
		return Id;
	}
};

struct OplogEntryAddress
{
	uint64_t Offset;
	uint64_t Size;
};

static_assert(IsPow2(sizeof(OplogEntry)));

/** Project Store
 */
class ProjectStore : public RefCounted, public GcContributor
{
	struct OplogStorage;

public:
	ProjectStore(CidStore& Store, std::filesystem::path BasePath, CasGc& Gc);
	~ProjectStore();

	struct Project;

	struct Oplog
	{
		Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath);
		~Oplog();

		[[nodiscard]] static bool ExistsAt(std::filesystem::path BasePath);

		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
		void IterateOplog(std::function<void(CbObject)>&& Fn);
		std::optional<CbObject> GetOplog(const Oid& Key);

		IoBuffer FindChunk(Oid ChunkId);

		inline static const uint32_t kInvalidOp = ~0u;

		/** Persist a new oplog entry
		 *
		 * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected
		 */
		uint32_t AppendNewOplogEntry(CbPackage Op);

		enum UpdateType
		{
			kUpdateNewEntry,
			kUpdateReplay
		};

		/** Update tracking metadata for a new oplog entry
		 *
		 * This is used during replay (and gets called as part of new op append)
		 *
		 * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected
		 */
		uint32_t RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry, UpdateType TypeOfUpdate);

		/** Scan oplog and register each entry, thus updating the in-memory tracking tables
		 */
		void ReplayLog();

		const std::string& OplogId() const { return m_OplogId; }

		const std::filesystem::path& TempPath() const { return m_TempPath; }

		spdlog::logger& Log() { return m_OuterProject->Log(); }
		void			Flush();
		void			Scrub(ScrubContext& Ctx) const;
		void			GatherReferences(GcContext& GcCtx);

		std::size_t OplogCount() const { return m_LatestOpMap.size(); }

	private:
		struct FileMapEntry
		{
			std::string ServerPath;
			std::string ClientPath;
		};

		template<class V>
		using OidMap = tsl::robin_map<Oid, V, Oid::Hasher>;

		Project*			  m_OuterProject = nullptr;
		CidStore&			  m_CidStore;
		std::filesystem::path m_BasePath;
		std::filesystem::path m_TempPath;

		mutable RwLock					 m_OplogLock;
		OidMap<IoHash>					 m_ChunkMap;		 // output data chunk id -> CAS address
		OidMap<IoHash>					 m_MetaMap;			 // meta chunk id -> CAS address
		OidMap<FileMapEntry>			 m_FileMap;			 // file id -> file map entry
		int32_t							 m_ManifestVersion;	 // File system manifest version
		std::map<int, OplogEntryAddress> m_OpAddressMap;	 // Index LSN -> op data in ops blob file
		OidMap<int>						 m_LatestOpMap;		 // op key -> latest op LSN for key

		RefPtr<OplogStorage> m_Storage;
		std::string			 m_OplogId;

		bool AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath);
		void AddChunkMapping(Oid ChunkId, IoHash Hash);
		void AddMetaMapping(Oid ChunkId, IoHash Hash);
	};

	struct Project
	{
		std::string			  Identifier;
		std::filesystem::path RootDir;
		std::string			  EngineRootDir;
		std::string			  ProjectRootDir;

		Oplog* NewOplog(std::string_view OplogId);
		Oplog* OpenOplog(std::string_view OplogId);
		void   DeleteOplog(std::string_view OplogId);
		void   IterateOplogs(std::function<void(const Oplog&)>&& Fn) const;
		void   IterateOplogs(std::function<void(Oplog&)>&& Fn);
		void   DiscoverOplogs();

		Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath);
		~Project();

		void					  Read();
		void					  Write();
		[[nodiscard]] static bool Exists(std::filesystem::path BasePath);
		void					  Flush();
		void					  Scrub(ScrubContext& Ctx);
		spdlog::logger&			  Log();
		void					  GatherReferences(GcContext& GcCtx);

	private:
		ProjectStore*				 m_ProjectStore;
		CidStore&					 m_CidStore;
		mutable RwLock				 m_ProjectLock;
		std::map<std::string, Oplog> m_Oplogs;
		std::filesystem::path		 m_OplogStoragePath;

		std::filesystem::path BasePathForOplog(std::string_view OplogId);
	};

	Oplog* OpenProjectOplog(std::string_view ProjectId, std::string_view OplogId);

	Project* OpenProject(std::string_view ProjectId);
	Project* NewProject(std::filesystem::path BasePath,
						std::string_view	  ProjectId,
						std::string_view	  RootDir,
						std::string_view	  EngineRootDir,
						std::string_view	  ProjectRootDir);
	void	 DeleteProject(std::string_view ProjectId);
	bool	 Exists(std::string_view ProjectId);
	void	 Flush();
	void	 Scrub(ScrubContext& Ctx);
	void	 DiscoverProjects();

	spdlog::logger&				 Log() { return m_Log; }
	const std::filesystem::path& BasePath() const { return m_ProjectBasePath; }

	virtual void GatherReferences(GcContext& GcCtx) override;

private:
	spdlog::logger&				   m_Log;
	CidStore&					   m_CidStore;
	std::filesystem::path		   m_ProjectBasePath;
	RwLock						   m_ProjectsLock;
	std::map<std::string, Project> m_Projects;

	std::filesystem::path BasePathForProject(std::string_view ProjectId);
};

//////////////////////////////////////////////////////////////////////////
//
//  {project}	a project identifier
//  {target}	a variation of the project, typically a build target
//  {lsn}		oplog entry sequence number
//
//  /prj/{project}
//  /prj/{project}/oplog/{target}
//  /prj/{project}/oplog/{target}/{lsn}
//
// oplog entry
//
// id: {id}
// key: {}
// meta: {}
// data: []
// refs:
//

class HttpProjectService : public HttpService
{
public:
	HttpProjectService(CidStore& Store, ProjectStore* InProjectStore);
	~HttpProjectService();

	virtual const char* BaseUri() const override;
	virtual void		HandleRequest(HttpServerRequest& Request) override;

private:
	inline spdlog::logger& Log() { return m_Log; }

	spdlog::logger&	  m_Log;
	CidStore&		  m_CidStore;
	HttpRequestRouter m_Router;
	Ref<ProjectStore> m_ProjectStore;
};

/** Project store interface for local clients
 *
 * This provides the same functionality as the HTTP interface but with
 * some optimizations which are only possible for clients running on the
 * same host as the Zen Store instance
 *
 */

class LocalProjectService : public RefCounted
{
protected:
	LocalProjectService(CasStore& CasStore, ProjectStore* Projects);
	~LocalProjectService();

public:
	static inline Ref<LocalProjectService> New(CasStore& Store, ProjectStore* Projects) { return new LocalProjectService(Store, Projects); }

private:
	struct LocalProjectImpl;

	CasStore&						  m_CasStore;
	Ref<ProjectStore>				  m_ProjectStore;
	std::unique_ptr<LocalProjectImpl> m_Impl;
};

void prj_forcelink();

}  // namespace zen