workspace shares (#84)

Feature: New 'workspaces' service which allows a user to share a local folder via zenserver. A workspace can have mulitple workspace shares and they provie an HTTP API that is compatible with the project oplog HTTP API. Workspaces and shares are preserved between runs. Workspaces feature is disabled by default - enable with --workspaces-enabled option when launching zenserver.

1 files changed, 389 insertions, 0 deletions

diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 4675ede38..15f863002 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -23,6 +23,7 @@
 #include <zenhttp/zenhttp.h>
 #include <zenutil/cache/cache.h>
 #include <zenutil/cache/cacherequests.h>
+#include <zenutil/chunkrequests.h>
 #include <zenutil/logging/testformatter.h>
 #include <zenutil/packageformat.h>
 #include <zenutil/zenserverprocess.h>
@@ -3196,6 +3197,394 @@ TEST_CASE("project.remote")
 	}
 }
 
+std::vector<std::pair<std::filesystem::path, IoBuffer>>
+GenerateFolderContent(const std::filesystem::path& RootPath)
+{
+	CreateDirectories(RootPath);
+	std::vector<std::pair<std::filesystem::path, IoBuffer>> Result;
+	Result.push_back(std::make_pair(RootPath / "root_blob_1.bin", CreateRandomBlob(4122)));
+	Result.push_back(std::make_pair(RootPath / "root_blob_2.bin", CreateRandomBlob(2122)));
+
+	std::filesystem::path EmptyFolder(RootPath / "empty_folder");
+
+	std::filesystem::path FirstFolder(RootPath / "first_folder");
+	std::filesystem::create_directory(FirstFolder);
+	Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22)));
+	Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122)));
+
+	std::filesystem::path SecondFolder(RootPath / "second_folder");
+	std::filesystem::create_directory(SecondFolder);
+	Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522)));
+	Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122)));
+	Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225)));
+
+	std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second");
+	std::filesystem::create_directory(SecondFolderChild);
+	Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622)));
+
+	for (const auto& It : Result)
+	{
+		WriteFile(It.first, It.second);
+	}
+
+	return Result;
+}
+
+std::vector<std::pair<std::filesystem::path, IoBuffer>>
+GenerateFolderContent2(const std::filesystem::path& RootPath)
+{
+	std::vector<std::pair<std::filesystem::path, IoBuffer>> Result;
+	Result.push_back(std::make_pair(RootPath / "root_blob_3.bin", CreateRandomBlob(312)));
+	std::filesystem::path FirstFolder(RootPath / "first_folder");
+	Result.push_back(std::make_pair(FirstFolder / "first_folder_blob3.bin", CreateRandomBlob(722)));
+	std::filesystem::path SecondFolder(RootPath / "second_folder");
+	std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second");
+	Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob2.bin", CreateRandomBlob(962)));
+	Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob3.bin", CreateRandomBlob(561)));
+
+	for (const auto& It : Result)
+	{
+		WriteFile(It.first, It.second);
+	}
+
+	return Result;
+}
+
+TEST_CASE("workspaces.create")
+{
+	using namespace std::literals;
+
+	std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir();
+
+	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+	ZenServerInstance	  Instance(TestEnv);
+	Instance.SetTestDir(TestDir);
+	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath));
+	CHECK(PortNumber != 0);
+
+	ScopedTemporaryDirectory TempDir;
+	std::filesystem::path	 Root1Path	= TempDir.Path() / "root1";
+	std::filesystem::path	 Root2Path	= TempDir.Path() / "root2";
+	std::filesystem::path	 Share1Path = "shared_1";
+	std::filesystem::path	 Share2Path = "shared_2";
+	CreateDirectories(Share1Path);
+	CreateDirectories(Share2Path);
+
+	Oid Root1Id = Oid::Zero;
+	Oid Root2Id = Oid::NewOid();
+
+	HttpClient Client(Instance.GetBaseUri());
+
+	CHECK(Client.Put(fmt::format("/ws/{}", Root1Id)).StatusCode == HttpResponseCode::BadRequest);
+
+	if (HttpClient::Response Root1Response =
+			Client.Put(fmt::format("/ws/{}", Oid::Zero), HttpClient::KeyValueMap{{"root_path", Root1Path.string()}});
+		Root1Response.StatusCode == HttpResponseCode::Created)
+	{
+		Root1Id = Oid::TryFromHexString(Root1Response.AsText());
+		CHECK(Root1Id != Oid::Zero);
+	}
+	else
+	{
+		CHECK(false);
+	}
+	if (HttpClient::Response Root1Response =
+			Client.Put(fmt::format("/ws/{}", Oid::Zero), HttpClient::KeyValueMap{{"root_path", Root1Path.string()}});
+		Root1Response.StatusCode == HttpResponseCode::OK)
+	{
+		CHECK(Root1Id == Oid::TryFromHexString(Root1Response.AsText()));
+	}
+	else
+	{
+		CHECK(false);
+	}
+	if (HttpClient::Response Root1Response =
+			Client.Put(fmt::format("/ws/{}", Root1Id), HttpClient::KeyValueMap{{"root_path", Root1Path.string()}});
+		Root1Response.StatusCode == HttpResponseCode::OK)
+	{
+		CHECK(Root1Id == Oid::TryFromHexString(Root1Response.AsText()));
+	}
+	else
+	{
+		CHECK(false);
+	}
+	CHECK(Client.Put(fmt::format("/ws/{}", Root1Id), HttpClient::KeyValueMap{{"root_path", Root2Path.string()}}).StatusCode ==
+		  HttpResponseCode::Conflict);
+
+	CHECK(
+		Client.Put(fmt::format("/ws/{}/{}", Root1Id, Oid::Zero), HttpClient::KeyValueMap{{"share_path", Share2Path.string()}}).StatusCode ==
+		HttpResponseCode::Created);
+
+	CHECK(
+		Client.Put(fmt::format("/ws/{}/{}", Root2Id, Oid::Zero), HttpClient::KeyValueMap{{"share_path", Share2Path.string()}}).StatusCode ==
+		HttpResponseCode::NotFound);
+
+	if (HttpClient::Response Root2Response =
+			Client.Put(fmt::format("/ws/{}", Root2Id), HttpClient::KeyValueMap{{"root_path", Root1Path.string()}});
+		Root2Response.StatusCode == HttpResponseCode::Created)
+	{
+		CHECK(Root2Id == Oid::TryFromHexString(Root2Response.AsText()));
+	}
+	else
+	{
+		CHECK(false);
+	}
+
+	CHECK(Client.Put(fmt::format("/ws/{}/{}", Root2Id, Oid::Zero)).StatusCode == HttpResponseCode::BadRequest);
+
+	Oid Share2Id = Oid::Zero;
+	if (HttpClient::Response Share2Response =
+			Client.Put(fmt::format("/ws/{}/{}", Root2Id, Share2Id), HttpClient::KeyValueMap{{"share_path", Share2Path.string()}});
+		Share2Response.StatusCode == HttpResponseCode::Created)
+	{
+		Share2Id = Oid::TryFromHexString(Share2Response.AsText());
+		CHECK(Share2Id != Oid::Zero);
+	}
+
+	CHECK(
+		Client.Put(fmt::format("/ws/{}/{}", Root2Id, Oid::Zero), HttpClient::KeyValueMap{{"share_path", Share2Path.string()}}).StatusCode ==
+		HttpResponseCode::OK);
+
+	CHECK(
+		Client.Put(fmt::format("/ws/{}/{}", Root2Id, Share2Id), HttpClient::KeyValueMap{{"share_path", Share2Path.string()}}).StatusCode ==
+		HttpResponseCode::OK);
+
+	CHECK(
+		Client.Put(fmt::format("/ws/{}/{}", Root2Id, Share2Id), HttpClient::KeyValueMap{{"share_path", Share1Path.string()}}).StatusCode ==
+		HttpResponseCode::Conflict);
+}
+
+TEST_CASE("workspaces.lifetimes")
+{
+	using namespace std::literals;
+
+	std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir();
+
+	Oid WorkspaceId = Oid::NewOid();
+	Oid ShareId		= Oid::NewOid();
+
+	{
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+		ZenServerInstance	  Instance(TestEnv);
+		Instance.SetTestDir(TestDir);
+		const uint16_t PortNumber =
+			Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath));
+		CHECK(PortNumber != 0);
+
+		ScopedTemporaryDirectory TempDir;
+		std::filesystem::path	 RootPath  = TempDir.Path();
+		std::filesystem::path	 SharePath = RootPath / "shared_folder";
+		CreateDirectories(SharePath);
+
+		HttpClient Client(Instance.GetBaseUri());
+		CHECK(Client.Put(fmt::format("/ws/{}", WorkspaceId), HttpClient::KeyValueMap{{"root_path", RootPath.string()}}).StatusCode ==
+			  HttpResponseCode::Created);
+		CHECK(Client.Get(fmt::format("/ws/{}", WorkspaceId)).AsObject()["id"sv].AsObjectId() == WorkspaceId);
+		CHECK(Client.Put(fmt::format("/ws/{}", WorkspaceId), HttpClient::KeyValueMap{{"root_path", RootPath.string()}}).StatusCode ==
+			  HttpResponseCode::OK);
+
+		CHECK(Client.Put(fmt::format("/ws/{}/{}", WorkspaceId, ShareId), HttpClient::KeyValueMap{{"share_path", SharePath.string()}})
+				  .StatusCode == HttpResponseCode::Created);
+		CHECK(Client.Get(fmt::format("/ws/{}/{}", WorkspaceId, ShareId)).AsObject()["id"sv].AsObjectId() == ShareId);
+		CHECK(Client.Put(fmt::format("/ws/{}/{}", WorkspaceId, ShareId), HttpClient::KeyValueMap{{"share_path", SharePath.string()}})
+				  .StatusCode == HttpResponseCode::OK);
+	}
+
+	// Restart
+
+	{
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+		ZenServerInstance	  Instance(TestEnv);
+		Instance.SetTestDir(TestDir);
+		const uint16_t PortNumber =
+			Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath));
+		CHECK(PortNumber != 0);
+
+		HttpClient Client(Instance.GetBaseUri());
+		CHECK(Client.Get(fmt::format("/ws/{}", WorkspaceId)).AsObject()["id"sv].AsObjectId() == WorkspaceId);
+
+		CHECK(Client.Get(fmt::format("/ws/{}/{}", WorkspaceId, ShareId)).AsObject()["id"sv].AsObjectId() == ShareId);
+	}
+
+	// Wipe system config
+	std::filesystem::remove_all(SystemRootPath);
+
+	// Restart
+
+	{
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+		ZenServerInstance	  Instance(TestEnv);
+		Instance.SetTestDir(TestDir);
+		const uint16_t PortNumber =
+			Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath));
+		CHECK(PortNumber != 0);
+
+		HttpClient Client(Instance.GetBaseUri());
+		CHECK(Client.Get(fmt::format("/ws/{}", WorkspaceId)).StatusCode == HttpResponseCode::NotFound);
+		CHECK(Client.Get(fmt::format("/ws/{}/{}", WorkspaceId, ShareId)).StatusCode == HttpResponseCode::NotFound);
+	}
+}
+
+TEST_CASE("workspaces.share")
+{
+	ZenServerInstance Instance(TestEnv);
+
+	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady("--workspaces-enabled");
+	CHECK(PortNumber != 0);
+
+	ScopedTemporaryDirectory TempDir;
+	std::filesystem::path	 RootPath  = TempDir.Path();
+	std::filesystem::path	 SharePath = RootPath / "shared_folder";
+	GenerateFolderContent(SharePath);
+
+	HttpClient Client(Instance.GetBaseUri());
+
+	Oid WorkspaceId = Oid::NewOid();
+	CHECK(Client.Put(fmt::format("/ws/{}", WorkspaceId), HttpClient::KeyValueMap{{"root_path", RootPath.string()}}).StatusCode ==
+		  HttpResponseCode::Created);
+	CHECK(Client.Get(fmt::format("/ws/{}", WorkspaceId)).AsObject()["id"sv].AsObjectId() == WorkspaceId);
+
+	Oid ShareId = Oid::NewOid();
+	CHECK(Client.Put(fmt::format("/ws/{}/{}", WorkspaceId, ShareId), HttpClient::KeyValueMap{{"share_path", SharePath.string()}})
+			  .StatusCode == HttpResponseCode::Created);
+	CHECK(Client.Get(fmt::format("/ws/{}/{}", WorkspaceId, ShareId)).AsObject()["id"sv].AsObjectId() == ShareId);
+
+	CHECK(Client.Get(fmt::format("/ws/{}/{}/files", WorkspaceId, ShareId)).AsObject()["files"sv].AsArrayView().Num() == 8);
+	GenerateFolderContent2(SharePath);
+	CHECK(Client.Get(fmt::format("/ws/{}/{}/files", WorkspaceId, ShareId)).AsObject()["files"sv].AsArrayView().Num() == 8);
+	HttpClient::Response FilesResponse =
+		Client.Get(fmt::format("/ws/{}/{}/files", WorkspaceId, ShareId),
+				   {},
+				   HttpClient::KeyValueMap{{"refresh", ToString(true)}, {"fieldnames", "id,clientpath,size"}});
+	CHECK(FilesResponse);
+	std::unordered_map<Oid, std::pair<std::filesystem::path, uint64_t>, Oid::Hasher> Files;
+	{
+		CbArrayView FilesArray = FilesResponse.AsObject()["files"sv].AsArrayView();
+		CHECK(FilesArray.Num() == 12);
+		for (CbFieldView Field : FilesArray)
+		{
+			CbObjectView FileObject = Field.AsObjectView();
+			Oid			 ChunkId	= FileObject["id"sv].AsObjectId();
+			CHECK(ChunkId != Oid::Zero);
+			uint64_t			  Size		  = FileObject["size"sv].AsUInt64();
+			std::u8string_view	  Path		  = FileObject["clientpath"sv].AsU8String();
+			std::filesystem::path AbsFilePath = SharePath / Path;
+			CHECK(std::filesystem::is_regular_file(AbsFilePath));
+			CHECK(std::filesystem::file_size(AbsFilePath) == Size);
+			Files.insert_or_assign(ChunkId, std::make_pair(AbsFilePath, Size));
+		}
+	}
+
+	HttpClient::Response EntriesResponse =
+		Client.Get(fmt::format("/ws/{}/{}/entries", WorkspaceId, ShareId), {}, HttpClient::KeyValueMap{{"fieldfilter", "id,clientpath"}});
+	CHECK(EntriesResponse);
+	{
+		CbArrayView EntriesArray = EntriesResponse.AsObject()["entries"sv].AsArrayView();
+		CHECK(EntriesArray.Num() == 1);
+		for (CbFieldView EntryField : EntriesArray)
+		{
+			CbObjectView EntryObject = EntryField.AsObjectView();
+			CbArrayView	 FilesArray	 = EntryObject["files"sv].AsArrayView();
+			CHECK(FilesArray.Num() == 12);
+			for (CbFieldView FileField : FilesArray)
+			{
+				CbObjectView FileObject = FileField.AsObjectView();
+				Oid			 ChunkId	= FileObject["id"sv].AsObjectId();
+				CHECK(ChunkId != Oid::Zero);
+				std::u8string_view	  Path		  = FileObject["clientpath"sv].AsU8String();
+				std::filesystem::path AbsFilePath = SharePath / Path;
+				CHECK(std::filesystem::is_regular_file(AbsFilePath));
+			}
+		}
+	}
+
+	HttpClient::Response FileManifestResponse =
+		Client.Get(fmt::format("/ws/{}/{}/entries", WorkspaceId, ShareId),
+				   {},
+				   HttpClient::KeyValueMap{{"opkey", "file_manifest"}, {"fieldfilter", "id,clientpath"}});
+	CHECK(FileManifestResponse);
+	{
+		CbArrayView EntriesArray = FileManifestResponse.AsObject()["entry"sv].AsObjectView()["files"sv].AsArrayView();
+		CHECK(EntriesArray.Num() == 12);
+		for (CbFieldView Field : EntriesArray)
+		{
+			CbObjectView FileObject = Field.AsObjectView();
+			Oid			 ChunkId	= FileObject["id"sv].AsObjectId();
+			CHECK(ChunkId != Oid::Zero);
+			std::u8string_view	  Path		  = FileObject["clientpath"sv].AsU8String();
+			std::filesystem::path AbsFilePath = SharePath / Path;
+			CHECK(std::filesystem::is_regular_file(AbsFilePath));
+		}
+	}
+
+	for (auto It : Files)
+	{
+		const Oid&					 ChunkId = It.first;
+		const std::filesystem::path& Path	 = It.second.first;
+		const uint64_t				 Size	 = It.second.second;
+
+		CHECK(Client.Get(fmt::format("/ws/{}/{}/{}/info", WorkspaceId, ShareId, ChunkId)).AsObject()["size"sv].AsUInt64() == Size);
+
+		{
+			IoBuffer Payload = Client.Get(fmt::format("/ws/{}/{}/{}", WorkspaceId, ShareId, ChunkId)).ResponsePayload;
+			CHECK(Payload);
+			CHECK(Payload.GetSize() == Size);
+			IoBuffer FileContent = IoBufferBuilder::MakeFromFile(Path);
+			CHECK(FileContent);
+			CHECK(FileContent.GetView().EqualBytes(Payload.GetView()));
+		}
+
+		{
+			IoBuffer Payload =
+				Client
+					.Get(fmt::format("/ws/{}/{}/{}", WorkspaceId, ShareId, ChunkId),
+						 {},
+						 HttpClient::KeyValueMap{{"offset", fmt::format("{}", Size / 4)}, {"size", fmt::format("{}", Size / 2)}})
+					.ResponsePayload;
+			CHECK(Payload);
+			CHECK(Payload.GetSize() == Size / 2);
+			IoBuffer FileContent = IoBufferBuilder::MakeFromFile(Path, Size / 4, Size / 2);
+			CHECK(FileContent);
+			CHECK(FileContent.GetView().EqualBytes(Payload.GetView()));
+		}
+	}
+
+	{
+		uint32_t					   CorrelationId = gsl::narrow<uint32_t>(Files.size());
+		std::vector<RequestChunkEntry> BatchEntries;
+		for (auto It : Files)
+		{
+			const Oid&	   ChunkId = It.first;
+			const uint64_t Size	   = It.second.second;
+
+			BatchEntries.push_back(
+				RequestChunkEntry{.ChunkId = ChunkId, .CorrelationId = --CorrelationId, .Offset = Size / 4, .RequestBytes = Size / 2});
+		}
+		IoBuffer BatchResponse =
+			Client.Post(fmt::format("/ws/{}/{}/batch", WorkspaceId, ShareId), BuildChunkBatchRequest(BatchEntries)).ResponsePayload;
+		CHECK(BatchResponse);
+		std::vector<IoBuffer> BatchResult = ParseChunkBatchResponse(BatchResponse);
+		CHECK(BatchResult.size() == Files.size());
+		for (const RequestChunkEntry& Request : BatchEntries)
+		{
+			IoBuffer					 Result = BatchResult[Request.CorrelationId];
+			auto						 It		= Files.find(Request.ChunkId);
+			const std::filesystem::path& Path	= It->second.first;
+			CHECK(Result.GetSize() == Request.RequestBytes);
+			IoBuffer FileContent = IoBufferBuilder::MakeFromFile(Path, Request.Offset, Request.RequestBytes);
+			CHECK(FileContent);
+			CHECK(FileContent.GetView().EqualBytes(Result.GetView()));
+		}
+	}
+
+	CHECK(Client.Delete(fmt::format("/ws/{}/{}", WorkspaceId, ShareId)));
+	CHECK(Client.Get(fmt::format("/ws/{}/{}", WorkspaceId, ShareId)).StatusCode == HttpResponseCode::NotFound);
+	CHECK(Client.Get(fmt::format("/ws/{}", WorkspaceId)));
+
+	CHECK(Client.Delete(fmt::format("/ws/{}", WorkspaceId)));
+	CHECK(Client.Get(fmt::format("/ws/{}", WorkspaceId)).StatusCode == HttpResponseCode::NotFound);
+}
+
 #	if 0
 TEST_CASE("lifetime.owner")
 {