aboutsummaryrefslogtreecommitdiff
path: root/zenserver
diff options
context:
space:
mode:
authorMartin Ridgers <[email protected]>2021-09-15 09:22:32 +0200
committerMartin Ridgers <[email protected]>2021-09-15 09:23:33 +0200
commit8f5e773529858223beeecf5d1b69c23991df644e (patch)
tree2c360c67e028f5ecd7368212b0adf8b23578ff9d /zenserver
parentUse zen::Sleep() in timer.cpp's tests (diff)
parentUpdated function service to new package management API (diff)
downloadzen-8f5e773529858223beeecf5d1b69c23991df644e.tar.xz
zen-8f5e773529858223beeecf5d1b69c23991df644e.zip
Merge main
Diffstat (limited to 'zenserver')
-rw-r--r--zenserver/admin/admin.h2
-rw-r--r--zenserver/cache/structuredcache.cpp63
-rw-r--r--zenserver/cache/structuredcache.h2
-rw-r--r--zenserver/casstore.cpp16
-rw-r--r--zenserver/casstore.h2
-rw-r--r--zenserver/compute/apply.cpp72
-rw-r--r--zenserver/compute/apply.h2
-rw-r--r--zenserver/diag/diagsvcs.h16
-rw-r--r--zenserver/diag/logging.cpp4
-rw-r--r--zenserver/projectstore.cpp123
-rw-r--r--zenserver/projectstore.h2
-rw-r--r--zenserver/sos/sos.h2
-rw-r--r--zenserver/testing/httptest.cpp106
-rw-r--r--zenserver/testing/httptest.h47
-rw-r--r--zenserver/testing/launch.cpp6
-rw-r--r--zenserver/testing/launch.h2
-rw-r--r--zenserver/upstream/jupiter.cpp2
-rw-r--r--zenserver/upstream/jupiter.h2
-rw-r--r--zenserver/vfs.cpp2
-rw-r--r--zenserver/vfs.h6
-rw-r--r--zenserver/xmake.lua5
-rw-r--r--zenserver/zenserver.cpp38
-rw-r--r--zenserver/zenserver.vcxproj8
-rw-r--r--zenserver/zenserver.vcxproj.filters5
24 files changed, 366 insertions, 169 deletions
diff --git a/zenserver/admin/admin.h b/zenserver/admin/admin.h
index 3bb8a9158..f90ad4537 100644
--- a/zenserver/admin/admin.h
+++ b/zenserver/admin/admin.h
@@ -2,7 +2,7 @@
#pragma once
-#include <zencore/httpserver.h>
+#include <zenhttp/httpserver.h>
class HttpAdminService : public zen::HttpService
{
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 0e235a9be..b2f8d191c 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -4,8 +4,8 @@
#include <zencore/compactbinaryvalidation.h>
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
-#include <zencore/httpserver.h>
#include <zencore/timer.h>
+#include <zenhttp/httpserver.h>
#include "structuredcache.h"
#include "structuredcachestore.h"
@@ -25,27 +25,6 @@ namespace zen {
using namespace std::literals;
-zen::HttpContentType
-MapToHttpContentType(zen::ZenContentType Type)
-{
- switch (Type)
- {
- default:
- case zen::ZenContentType::kBinary:
- return zen::HttpContentType::kBinary;
- case zen::ZenContentType::kCbObject:
- return zen::HttpContentType::kCbObject;
- case zen::ZenContentType::kCbPackage:
- return zen::HttpContentType::kCbPackage;
- case zen::ZenContentType::kText:
- return zen::HttpContentType::kText;
- case zen::ZenContentType::kJSON:
- return zen::HttpContentType::kJSON;
- case zen::ZenContentType::kYAML:
- return zen::HttpContentType::kYAML;
- }
-};
-
//////////////////////////////////////////////////////////////////////////
HttpStructuredCacheService::HttpStructuredCacheService(::ZenCacheStore& InCacheStore,
@@ -93,7 +72,7 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request)
return HandleCacheBucketRequest(Request, Key);
}
- return Request.WriteResponse(zen::HttpResponse::BadRequest); // invalid URL
+ return Request.WriteResponse(zen::HttpResponseCode::BadRequest); // invalid URL
}
if (Ref.PayloadId == IoHash::Zero)
@@ -128,11 +107,11 @@ HttpStructuredCacheService::HandleCacheBucketRequest(zen::HttpServerRequest& Req
if (m_CacheStore.DropBucket(Bucket))
{
- return Request.WriteResponse(zen::HttpResponse::OK);
+ return Request.WriteResponse(zen::HttpResponseCode::OK);
}
else
{
- return Request.WriteResponse(zen::HttpResponse::NotFound);
+ return Request.WriteResponse(zen::HttpResponseCode::NotFound);
}
break;
}
@@ -209,7 +188,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
{
m_Log.debug("MISS - '{}/{}'", Ref.BucketSegment, Ref.HashKey);
- return Request.WriteResponse(zen::HttpResponse::NotFound);
+ return Request.WriteResponse(zen::HttpResponseCode::NotFound);
}
if (Verb == kHead)
@@ -224,7 +203,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
Value.Value.GetContentType(),
InUpstreamCache ? "upstream" : "local");
- return Request.WriteResponse(zen::HttpResponse::OK, MapToHttpContentType(Value.Value.GetContentType()), Value.Value);
+ return Request.WriteResponse(zen::HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value);
}
break;
@@ -234,7 +213,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
if (!Body || Body.Size() == 0)
{
- return Request.WriteResponse(zen::HttpResponse::BadRequest);
+ return Request.WriteResponse(zen::HttpResponseCode::BadRequest);
}
const HttpContentType ContentType = Request.RequestContentType();
@@ -253,7 +232,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
break;
default:
- return Request.WriteResponse(zen::HttpResponse::BadRequest);
+ return Request.WriteResponse(zen::HttpResponseCode::BadRequest);
}
if (!IsCompactBinary)
@@ -272,7 +251,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
{.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}});
}
- return Request.WriteResponse(zen::HttpResponse::Created);
+ return Request.WriteResponse(zen::HttpResponseCode::Created);
}
// Validate payload before accessing it
@@ -284,7 +263,9 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
m_Log.warn("Payload for key '{}/{}' ({} bytes) failed validation", Ref.BucketSegment, Ref.HashKey, Body.Size());
// TODO: add details in response, kText || kCbObject?
- return Request.WriteResponse(HttpResponse::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv);
+ return Request.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Compact binary validation failed"sv);
}
// Extract referenced payload hashes
@@ -336,7 +317,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
.PayloadIds = std::move(References)});
}
- return Request.WriteResponse(zen::HttpResponse::Created);
+ return Request.WriteResponse(zen::HttpResponseCode::Created);
}
else
{
@@ -351,7 +332,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
Response.EndArray();
// Return Created | BadRequest?
- return Request.WriteResponse(zen::HttpResponse::Created, Response.Save());
+ return Request.WriteResponse(zen::HttpResponseCode::Created, Response.Save());
}
}
break;
@@ -407,7 +388,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
if (!Payload)
{
m_Log.debug("MISS - '{}/{}/{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId);
- return Request.WriteResponse(zen::HttpResponse::NotFound);
+ return Request.WriteResponse(zen::HttpResponseCode::NotFound);
}
m_Log.debug("HIT - '{}/{}/{}' ({} bytes, {}) ({})",
@@ -423,7 +404,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
Request.SetSuppressResponseBody();
}
- return Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Payload);
+ return Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Payload);
}
break;
@@ -433,7 +414,9 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
{
if (Body.Size() == 0)
{
- return Request.WriteResponse(zen::HttpResponse::BadRequest, HttpContentType::kText, "Empty payload not permitted");
+ return Request.WriteResponse(zen::HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Empty payload not permitted");
}
zen::IoHash ChunkHash = zen::IoHash::HashBuffer(Body);
@@ -443,7 +426,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
if (!Compressed)
{
// All attachment payloads need to be in compressed buffer format
- return Request.WriteResponse(zen::HttpResponse::BadRequest,
+ return Request.WriteResponse(zen::HttpResponseCode::BadRequest,
HttpContentType::kText,
"Attachments must be compressed");
}
@@ -452,7 +435,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Ref.PayloadId)
{
// the URL specified content id and content hashes don't match!
- return Request.WriteResponse(HttpResponse::BadRequest);
+ return Request.WriteResponse(HttpResponseCode::BadRequest);
}
zen::CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash);
@@ -469,11 +452,11 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
if (Result.New)
{
- return Request.WriteResponse(zen::HttpResponse::Created);
+ return Request.WriteResponse(zen::HttpResponseCode::Created);
}
else
{
- return Request.WriteResponse(zen::HttpResponse::OK);
+ return Request.WriteResponse(zen::HttpResponseCode::OK);
}
}
}
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index b90301d84..d4bb94c52 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -2,7 +2,7 @@
#pragma once
-#include <zencore/httpserver.h>
+#include <zenhttp/httpserver.h>
#include <spdlog/spdlog.h>
#include <memory>
diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp
index 1d147024a..6f1e4873b 100644
--- a/zenserver/casstore.cpp
+++ b/zenserver/casstore.cpp
@@ -23,7 +23,7 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store)
if ((EntryCount * sizeof(IoHash)) != Payload.Size())
{
- return ServerRequest.WriteResponse(HttpResponse::BadRequest);
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest);
}
const IoHash* Hashes = reinterpret_cast<const IoHash*>(Payload.Data());
@@ -55,7 +55,7 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store)
Values[0] = IoBufferBuilder::MakeCloneFromMemory(HeaderStream.Data(), HeaderStream.Size());
- ServerRequest.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Values);
+ ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Values);
},
HttpVerb::kPost);
@@ -74,10 +74,10 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store)
{
if (IoBuffer Value = m_CasStore.FindChunk(Hash))
{
- return ServerRequest.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Value);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value);
}
- return ServerRequest.WriteResponse(HttpResponse::NotFound);
+ return ServerRequest.WriteResponse(HttpResponseCode::NotFound);
}
break;
@@ -89,12 +89,12 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store)
// URI hash must match content hash
if (PayloadHash != Hash)
{
- return ServerRequest.WriteResponse(HttpResponse::BadRequest);
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest);
}
m_CasStore.InsertChunk(Payload, PayloadHash);
- return ServerRequest.WriteResponse(HttpResponse::OK);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK);
}
break;
}
@@ -129,11 +129,11 @@ HttpCasService::HandleRequest(zen::HttpServerRequest& Request)
if (InsertResult.New)
{
- return Request.WriteResponse(HttpResponse::Created);
+ return Request.WriteResponse(HttpResponseCode::Created);
}
else
{
- return Request.WriteResponse(HttpResponse::OK);
+ return Request.WriteResponse(HttpResponseCode::OK);
}
}
break;
diff --git a/zenserver/casstore.h b/zenserver/casstore.h
index 7166f796e..4ca6908b5 100644
--- a/zenserver/casstore.h
+++ b/zenserver/casstore.h
@@ -2,7 +2,7 @@
#pragma once
-#include <zencore/httpserver.h>
+#include <zenhttp/httpserver.h>
#include <zenstore/cas.h>
namespace zen {
diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp
index 939ac3362..94dedf087 100644
--- a/zenserver/compute/apply.cpp
+++ b/zenserver/compute/apply.cpp
@@ -351,12 +351,12 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
if (auto It = m_WorkerMap.find(WorkerId); It == m_WorkerMap.end())
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
else
{
const WorkerDesc& Desc = It->second;
- return HttpReq.WriteResponse(HttpResponse::OK, Desc.Descriptor);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Desc.Descriptor);
}
}
break;
@@ -378,6 +378,10 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
ChunkSet.AddChunk(Hash);
});
+ // Note that we store executables uncompressed to make it
+ // more straightforward and efficient to materialize them, hence
+ // the CAS lookup here instead of CID for the input payloads
+
m_CasStore.FilterChunks(ChunkSet);
if (ChunkSet.IsEmpty())
@@ -388,7 +392,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
spdlog::debug("worker {}: all attachments already available", WorkerId);
- return HttpReq.WriteResponse(HttpResponse::NoContent);
+ return HttpReq.WriteResponse(HttpResponseCode::NoContent);
}
else
{
@@ -406,7 +410,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
spdlog::debug("worker {}: need {} attachments", WorkerId, ChunkSet.GetChunkSet().size());
- return HttpReq.WriteResponse(HttpResponse::NotFound, ResponseWriter.Save());
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound, ResponseWriter.Save());
}
}
break;
@@ -426,21 +430,25 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
for (const CbAttachment& Attachment : Attachments)
{
- ZEN_ASSERT(Attachment.IsBinary());
+ ZEN_ASSERT(Attachment.IsCompressedBinary());
- const IoHash DataHash = Attachment.GetHash();
- CompressedBuffer DataView = Attachment.AsCompressedBinary();
+ const IoHash DataHash = Attachment.GetHash();
+ CompressedBuffer DataView = Attachment.AsCompressedBinary();
+ SharedBuffer Decompressed = DataView.Decompress();
+ const uint64_t DecompressedSize = DataView.GetRawSize();
- TotalAttachmentBytes += DataView.GetCompressedSize();
+ TotalAttachmentBytes += DecompressedSize;
++AttachmentCount;
- IoBuffer Payload = DataView.GetCompressed().Flatten().AsIoBuffer();
+ // Note that we store executables uncompressed to make it
+ // more straightforward and efficient to materialize them
- CasStore::InsertResult InsertResult = m_CasStore.InsertChunk(Payload, DataHash);
+ const CasStore::InsertResult InsertResult =
+ m_CasStore.InsertChunk(Decompressed.AsIoBuffer(), IoHash::FromBLAKE3(DataView.GetRawHash()));
if (InsertResult.New)
{
- TotalNewBytes += Payload.Size();
+ TotalNewBytes += DecompressedSize;
++NewAttachmentCount;
}
}
@@ -456,7 +464,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
m_WorkerMap.insert_or_assign(WorkerId, WorkerDesc{.Descriptor = Obj});
- return HttpReq.WriteResponse(HttpResponse::NoContent);
+ return HttpReq.WriteResponse(HttpResponseCode::NoContent);
}
break;
}
@@ -495,7 +503,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
if (auto It = m_WorkerMap.find(WorkerId); It == m_WorkerMap.end())
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
else
{
@@ -526,7 +534,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
RequestObject.IterateAttachments([&](CbFieldView Field) {
const IoHash FileHash = Field.AsHash();
- if (!m_CasStore.FindChunk(FileHash))
+ if (!m_CidStore.ContainsChunk(FileHash))
{
NeedList.push_back(FileHash);
}
@@ -538,7 +546,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
CbPackage Output = ExecAction(Worker, RequestObject);
- return HttpReq.WriteResponse(HttpResponse::OK, Output);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Output);
}
CbObjectWriter Cbo;
@@ -552,7 +560,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
Cbo.EndArray();
CbObject Response = Cbo.Save();
- return HttpReq.WriteResponse(HttpResponse::NotFound, Response);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound, Response);
}
break;
@@ -570,19 +578,21 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
for (const CbAttachment& Attachment : Attachments)
{
- ZEN_ASSERT(Attachment.IsBinary());
+ ZEN_ASSERT(Attachment.IsCompressedBinary());
+
+ const IoHash DataHash = Attachment.GetHash();
+ CompressedBuffer DataView = Attachment.AsCompressedBinary();
- const IoHash DataHash = Attachment.GetHash();
- SharedBuffer DataView = Attachment.AsBinary();
+ const uint64_t CompressedSize = DataView.GetCompressedSize();
- TotalAttachmentBytes += DataView.GetSize();
+ TotalAttachmentBytes += CompressedSize;
++AttachmentCount;
- CasStore::InsertResult InsertResult = m_CasStore.InsertChunk(DataView.AsIoBuffer(), DataHash);
+ const CasStore::InsertResult InsertResult = m_CidStore.AddChunk(DataView);
if (InsertResult.New)
{
- TotalNewBytes += DataView.GetSize();
+ TotalNewBytes += CompressedSize;
++NewAttachmentCount;
}
}
@@ -595,7 +605,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
CbPackage Output = ExecAction(Worker, ActionObj);
- return HttpReq.WriteResponse(HttpResponse::OK, Output);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Output);
}
break;
}
@@ -659,7 +669,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
if (!AllOk)
{
// TODO: Could report all the missing pieces in the response here
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
std::string Executable8{RequestObject["cmd"].AsString()};
@@ -681,7 +691,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
Response << "exitcode" << Job.ExitCode();
- return HttpReq.WriteResponse(HttpResponse::OK, Response.Save());
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
}
break;
}
@@ -780,18 +790,16 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action)
// Manifest inputs in sandbox
Action.IterateAttachments([&](CbFieldView Field) {
- const IoHash Hash = Field.AsHash();
- std::filesystem::path FilePath{SandboxPath / "Inputs" / Hash.ToHexString()};
- IoBuffer DataBuffer = m_CasStore.FindChunk(Hash);
+ const IoHash Cid = Field.AsHash();
+ std::filesystem::path FilePath{SandboxPath / "Inputs" / Cid.ToHexString()};
+ IoBuffer DataBuffer = m_CidStore.FindChunkByCid(Cid);
if (!DataBuffer)
{
throw std::exception("Chunk missing" /* ADD CONTEXT */);
}
- CompressedBuffer Buffer = CompressedBuffer::Compress(SharedBuffer(std::move(DataBuffer)));
-
- zen::WriteFile(FilePath, Buffer.GetCompressed().Flatten().AsIoBuffer());
+ zen::WriteFile(FilePath, DataBuffer);
});
// Set up environment variables
@@ -884,7 +892,7 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action)
ZEN_ASSERT(OutputData.Data.size() == 1);
- CbAttachment Attachment(SharedBuffer(ChunkData.Data[0]), Hash);
+ CbAttachment Attachment(CompressedBuffer::FromCompressed(SharedBuffer(ChunkData.Data[0])));
OutputPackage.AddAttachment(Attachment);
});
diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h
index 20a58507b..695dc2e6e 100644
--- a/zenserver/compute/apply.h
+++ b/zenserver/compute/apply.h
@@ -3,9 +3,9 @@
#pragma once
#include <zencore/compactbinary.h>
-#include <zencore/httpserver.h>
#include <zencore/iohash.h>
#include <zencore/logging.h>
+#include <zenhttp/httpserver.h>
#include <filesystem>
#include <unordered_map>
diff --git a/zenserver/diag/diagsvcs.h b/zenserver/diag/diagsvcs.h
index 84f8d22ee..51ee98f67 100644
--- a/zenserver/diag/diagsvcs.h
+++ b/zenserver/diag/diagsvcs.h
@@ -2,8 +2,8 @@
#pragma once
-#include <zencore/httpserver.h>
#include <zencore/iobuffer.h>
+#include <zenhttp/httpserver.h>
//////////////////////////////////////////////////////////////////////////
@@ -25,17 +25,17 @@ public:
if (Uri == "hello"sv)
{
- Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kText, u8"hello world!"sv);
+ Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kText, u8"hello world!"sv);
// OutputLogMessageInternal(&LogPoint, 0, 0);
}
else if (Uri == "1K"sv)
{
- Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, m_1k);
+ Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, m_1k);
}
else if (Uri == "1M"sv)
{
- Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, m_1m);
+ Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, m_1m);
}
else if (Uri == "1M_1k"sv)
{
@@ -47,7 +47,7 @@ public:
Buffers.push_back(m_1k);
}
- Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers);
+ Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Buffers);
}
else if (Uri == "1G"sv)
{
@@ -59,7 +59,7 @@ public:
Buffers.push_back(m_1m);
}
- Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers);
+ Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Buffers);
}
else if (Uri == "1G_1k"sv)
{
@@ -71,7 +71,7 @@ public:
Buffers.push_back(m_1k);
}
- Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers);
+ Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Buffers);
}
}
@@ -95,7 +95,7 @@ public:
switch (Request.RequestVerb())
{
case zen::HttpVerb::kGet:
- return Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kText, u8"OK!"sv);
+ return Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kText, u8"OK!"sv);
}
}
diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp
index 796a15d01..5782ce582 100644
--- a/zenserver/diag/logging.cpp
+++ b/zenserver/diag/logging.cpp
@@ -195,13 +195,13 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
std::filesystem::path LogPath = GlobalOptions.DataDir / "logs/zenserver.log";
- bool IsAsync = true;
+ bool IsAsync = true;
spdlog::level::level_enum LogLevel = spdlog::level::info;
if (GlobalOptions.IsDebug)
{
LogLevel = spdlog::level::debug;
- IsAsync = false;
+ IsAsync = false;
}
if (IsAsync)
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 006796b28..2bbc1dce3 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -542,8 +542,30 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage)
for (const auto& Attach : Attachments)
{
- IoBuffer AttachmentData = Attach.AsBinary().AsIoBuffer();
- CasStore::InsertResult Result = m_CasStore.InsertChunk(AttachmentData, Attach.GetHash());
+ IoBuffer AttachmentData;
+
+ if (Attach.IsBinary())
+ {
+ AttachmentData = Attach.AsBinary().AsIoBuffer();
+ }
+ else if (Attach.IsCompressedBinary())
+ {
+ ZEN_NOT_IMPLEMENTED("Compressed binary attachments are currently not supported for oplogs");
+
+ AttachmentData = Attach.AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer();
+ }
+ else if (Attach.IsObject())
+ {
+ AttachmentData = Attach.AsObject().GetBuffer().AsIoBuffer();
+ }
+ else
+ {
+ ZEN_NOT_IMPLEMENTED("Unknown attachment type");
+ }
+
+ ZEN_ASSERT(AttachmentData);
+
+ CasStore::InsertResult Result = m_CasStore.InsertChunk(AttachmentData, Attach.GetHash());
const uint64_t AttachmentSize = AttachmentData.Size();
@@ -909,7 +931,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (FoundLog == nullptr)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
// Parse Request
@@ -940,7 +962,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (Payload.Size() <= sizeof(RequestHeader))
{
- HttpReq.WriteResponse(HttpResponse::BadRequest);
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest);
}
RequestHeader RequestHdr;
@@ -948,7 +970,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (RequestHdr.Magic != RequestHeader::kMagic)
{
- HttpReq.WriteResponse(HttpResponse::BadRequest);
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest);
}
std::vector<RequestChunkEntry> RequestedChunks;
@@ -1004,9 +1026,9 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
ResponsePtr += sizeof(ResponseHdr);
for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex)
{
- //const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
- const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]);
- ResponseChunkEntry ResponseChunk;
+ // const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex];
+ const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]);
+ ResponseChunkEntry ResponseChunk;
ResponseChunk.CorrelationId = ChunkIndex;
if (FoundChunk)
{
@@ -1020,7 +1042,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
ResponsePtr += sizeof(ResponseChunk);
}
- return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, OutBlobs);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs);
},
HttpVerb::kPost);
@@ -1038,7 +1060,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (FoundLog == nullptr)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
ProjectStore::Oplog& Log = *FoundLog;
@@ -1063,7 +1085,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
Response.EndArray();
- return HttpReq.WriteResponse(HttpResponse::OK, Response.Save());
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
},
HttpVerb::kGet);
@@ -1080,7 +1102,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (FoundLog == nullptr)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
ProjectStore::Oplog& Log = *FoundLog;
@@ -1093,10 +1115,10 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
{
CbObjectWriter Response;
Response << "size" << Value.Size();
- return HttpReq.WriteResponse(HttpResponse::OK, Response.Save());
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
}
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
},
HttpVerb::kGet);
@@ -1124,7 +1146,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
}
else
{
- return HttpReq.WriteResponse(HttpResponse::BadRequest);
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
}
}
@@ -1137,7 +1159,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
}
else
{
- return HttpReq.WriteResponse(HttpResponse::BadRequest);
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
}
}
@@ -1147,7 +1169,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (FoundLog == nullptr)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
ProjectStore::Oplog& Log = *FoundLog;
@@ -1162,7 +1184,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
case HttpVerb::kGet:
if (!Value)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
if (Verb == HttpVerb::kHead)
@@ -1185,10 +1207,10 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
// Send only a subset of data
IoBuffer InnerValue(Value, Offset, Size);
- return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, InnerValue);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, InnerValue);
}
- return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Value);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value);
}
},
HttpVerb::kGet | HttpVerb::kHead);
@@ -1217,7 +1239,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
}
else
{
- return HttpReq.WriteResponse(HttpResponse::BadRequest);
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
}
}
@@ -1230,7 +1252,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
}
else
{
- return HttpReq.WriteResponse(HttpResponse::BadRequest);
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
}
}
@@ -1240,7 +1262,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
IoBuffer Value = m_CasStore.FindChunk(Hash);
if (!Value)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
if (IsOffset)
@@ -1258,10 +1280,10 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
// Send only a subset of data
IoBuffer InnerValue(Value, Offset, Size);
- return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, InnerValue);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, InnerValue);
}
- return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Value);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value);
},
HttpVerb::kGet);
@@ -1277,7 +1299,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (FoundLog == nullptr)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
// This operation takes a list of referenced hashes and decides which
@@ -1312,7 +1334,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
Cbo.EndArray();
CbObject Response = Cbo.Save();
- return HttpReq.WriteResponse(HttpResponse::OK, Response);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
},
HttpVerb::kPost);
@@ -1340,7 +1362,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (FoundLog == nullptr)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
ProjectStore::Oplog& Log = *FoundLog;
@@ -1387,23 +1409,26 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
};
CbPackage Package;
- if (!Package.TryLoad(Payload, &UniqueBuffer::Alloc, &Resolver))
+
+ if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver))
{
- return HttpReq.WriteResponse(HttpResponse::BadRequest, HttpContentType::kText, "Invalid package");
+ m_Log.error("Received malformed package!");
+
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package");
}
if (!IsValid)
{
// TODO: emit diagnostics identifying missing chunks
- return HttpReq.WriteResponse(HttpResponse::NotFound, HttpContentType::kText, "Missing chunk reference");
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing chunk reference");
}
CbObject Core = Package.GetObject();
if (!Core["key"sv])
{
- return HttpReq.WriteResponse(HttpResponse::BadRequest, HttpContentType::kText, "No oplog entry key specified");
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
}
// Write core to oplog
@@ -1412,12 +1437,12 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (OpLsn == ProjectStore::Oplog::kInvalidOp)
{
- return HttpReq.WriteResponse(HttpResponse::BadRequest);
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
}
m_Log.info("new op #{:4} - {}/{} ({:>6}) {}", OpLsn, ProjectId, OplogId, NiceBytes(Payload.Size()), Core["key"sv].AsString());
- HttpReq.WriteResponse(HttpResponse::Created);
+ HttpReq.WriteResponse(HttpResponseCode::Created);
},
HttpVerb::kPost);
@@ -1428,7 +1453,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
// TODO: look up op and respond with the payload!
- HttpReq.WriteResponse(HttpResponse::Accepted, HttpContentType::kText, u8"yeee"sv);
+ HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, u8"yeee"sv);
},
HttpVerb::kGet);
@@ -1444,7 +1469,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (!ProjectIt)
{
- return Req.ServerRequest().WriteResponse(HttpResponse::NotFound,
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
HttpContentType::kText,
"project {} not found"_format(ProjectId));
}
@@ -1459,7 +1484,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (!OplogIt)
{
- return Req.ServerRequest().WriteResponse(HttpResponse::NotFound,
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
HttpContentType::kText,
"oplog {} not found in project {}"_format(OplogId, ProjectId));
}
@@ -1469,7 +1494,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
CbObjectWriter Cb;
Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempDir();
- Req.ServerRequest().WriteResponse(HttpResponse::OK, Cb.Save());
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save());
}
break;
@@ -1482,18 +1507,18 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (!Prj.NewOplog(OplogId))
{
// TODO: indicate why the operation failed!
- return Req.ServerRequest().WriteResponse(HttpResponse::InternalServerError);
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError);
}
m_Log.info("established oplog {} / {}", ProjectId, OplogId);
- return Req.ServerRequest().WriteResponse(HttpResponse::Created);
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::Created);
}
// I guess this should ultimately be used to execute RPCs but for now, it
// does absolutely nothing
- return Req.ServerRequest().WriteResponse(HttpResponse::BadRequest);
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
}
break;
@@ -1503,7 +1528,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
ProjectIt->DeleteOplog(OplogId);
- return Req.ServerRequest().WriteResponse(HttpResponse::OK);
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
}
break;
}
@@ -1522,7 +1547,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (FoundLog == nullptr)
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
CbObjectWriter Response;
@@ -1542,7 +1567,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
}
else
{
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
}
else
@@ -1555,7 +1580,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
}
}
- return HttpReq.WriteResponse(HttpResponse::OK, Response.Save());
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
},
HttpVerb::kGet);
@@ -1585,7 +1610,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
EngineRoot,
ProjectRoot);
- Req.ServerRequest().WriteResponse(HttpResponse::Created);
+ Req.ServerRequest().WriteResponse(HttpResponseCode::Created);
}
break;
@@ -1595,7 +1620,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (!ProjectIt)
{
- return Req.ServerRequest().WriteResponse(HttpResponse::NotFound,
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
HttpContentType::kText,
"project {} not found"_format(ProjectId));
}
@@ -1609,7 +1634,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
Prj.IterateOplogs([&](const ProjectStore::Oplog& I) { Response << "id"sv << I.OplogId(); });
Response.EndArray(); // oplogs
- Req.ServerRequest().WriteResponse(HttpResponse::OK, Response.Save());
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
}
break;
@@ -1619,7 +1644,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
if (!ProjectIt)
{
- return Req.ServerRequest().WriteResponse(HttpResponse::NotFound,
+ return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
HttpContentType::kText,
"project {} not found"_format(ProjectId));
}
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index 33bacfdaa..8fe189ab9 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -2,9 +2,9 @@
#pragma once
-#include <zencore/httpserver.h>
#include <zencore/uid.h>
#include <zencore/xxhash.h>
+#include <zenhttp/httpserver.h>
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
diff --git a/zenserver/sos/sos.h b/zenserver/sos/sos.h
index 283735cbd..da9064262 100644
--- a/zenserver/sos/sos.h
+++ b/zenserver/sos/sos.h
@@ -2,7 +2,7 @@
#pragma once
-#include <zencore/httpserver.h>
+#include <zenhttp/httpserver.h>
#include <spdlog/spdlog.h>
diff --git a/zenserver/testing/httptest.cpp b/zenserver/testing/httptest.cpp
new file mode 100644
index 000000000..c4fd6003c
--- /dev/null
+++ b/zenserver/testing/httptest.cpp
@@ -0,0 +1,106 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "httptest.h"
+
+#include <zencore/compactbinarypackage.h>
+
+namespace zen {
+
+HttpTestingService::HttpTestingService()
+{
+ m_Router.RegisterRoute(
+ "hello",
+ [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "echo",
+ [this](HttpRouterRequest& Req) {
+ IoBuffer Body = Req.ServerRequest().ReadPayload();
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Body);
+ },
+ HttpVerb::kPost);
+
+ m_Router.RegisterRoute(
+ "package",
+ [this](HttpRouterRequest& Req) {
+ CbPackage Pkg = Req.ServerRequest().ReadPayloadPackage();
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Pkg);
+ },
+ HttpVerb::kPost);
+}
+
+HttpTestingService::~HttpTestingService()
+{
+}
+
+const char*
+HttpTestingService::BaseUri() const
+{
+ return "/testing/";
+}
+
+void
+HttpTestingService::HandleRequest(HttpServerRequest& Request)
+{
+ m_Router.HandleRequest(Request);
+}
+
+Ref<IHttpPackageHandler>
+HttpTestingService::HandlePackageRequest(HttpServerRequest& HttpServiceRequest)
+{
+ RwLock::ExclusiveLockScope _(m_RwLock);
+
+ const uint32_t RequestId = HttpServiceRequest.RequestId();
+
+ if (auto It = m_HandlerMap.find(RequestId); It != m_HandlerMap.end())
+ {
+ Ref<HttpTestingService::PackageHandler> Handler = std::move(It->second);
+
+ m_HandlerMap.erase(It);
+
+ return Handler.Get();
+ }
+
+ auto InsertResult = m_HandlerMap.insert({RequestId, nullptr});
+
+ _.ReleaseNow();
+
+ return (InsertResult.first->second = new PackageHandler(*this, RequestId)).Get();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+HttpTestingService::PackageHandler::PackageHandler(HttpTestingService& Svc, uint32_t RequestId) : m_Svc(Svc), m_RequestId(RequestId)
+{
+}
+
+HttpTestingService::PackageHandler::~PackageHandler()
+{
+}
+
+void
+HttpTestingService::PackageHandler::FilterOffer(std::vector<IoHash>& OfferCids)
+{
+ ZEN_UNUSED(OfferCids);
+ // No-op
+ return;
+}
+void
+HttpTestingService::PackageHandler::OnRequestBegin()
+{
+}
+
+void
+HttpTestingService::PackageHandler::OnRequestComplete()
+{
+}
+
+IoBuffer
+HttpTestingService::PackageHandler::CreateTarget(const IoHash& Cid, uint64_t StorageSize)
+{
+ ZEN_UNUSED(Cid);
+ return IoBuffer{StorageSize};
+}
+
+} // namespace zen
diff --git a/zenserver/testing/httptest.h b/zenserver/testing/httptest.h
new file mode 100644
index 000000000..5809d4e2e
--- /dev/null
+++ b/zenserver/testing/httptest.h
@@ -0,0 +1,47 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenhttp/httpserver.h>
+
+#include <spdlog/spdlog.h>
+
+namespace zen {
+
+/**
+ * Test service to facilitate testing the HTTP framework and client interactions
+ */
+class HttpTestingService : public HttpService
+{
+public:
+ HttpTestingService();
+ ~HttpTestingService();
+
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(HttpServerRequest& Request) override;
+ virtual Ref<IHttpPackageHandler> HandlePackageRequest(HttpServerRequest& HttpServiceRequest) override;
+
+ class PackageHandler : public IHttpPackageHandler
+ {
+ public:
+ PackageHandler(HttpTestingService& Svc, uint32_t RequestId);
+ ~PackageHandler();
+
+ virtual void FilterOffer(std::vector<IoHash>& OfferCids) override;
+ virtual void OnRequestBegin() override;
+ virtual IoBuffer CreateTarget(const IoHash& Cid, uint64_t StorageSize) override;
+ virtual void OnRequestComplete() override;
+
+ private:
+ HttpTestingService& m_Svc;
+ uint32_t m_RequestId;
+ };
+
+private:
+ HttpRequestRouter m_Router;
+
+ RwLock m_RwLock;
+ std::unordered_map<uint32_t, Ref<PackageHandler>> m_HandlerMap;
+};
+
+} // namespace zen
diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp
index d06fae3e2..b031193d5 100644
--- a/zenserver/testing/launch.cpp
+++ b/zenserver/testing/launch.cpp
@@ -409,7 +409,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat
Cbo.EndArray();
CbObject Response = Cbo.Save();
- return HttpReq.WriteResponse(HttpResponse::OK, Response);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
}
break;
}
@@ -470,7 +470,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat
if (!AllOk)
{
// TODO: Could report all the missing pieces in the response here
- return HttpReq.WriteResponse(HttpResponse::NotFound);
+ return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
std::string Executable8{RequestObject["cmd"].AsString()};
@@ -492,7 +492,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat
Response << "exitcode" << Job.ExitCode();
- return HttpReq.WriteResponse(HttpResponse::OK, Response.Save());
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
}
break;
}
diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h
index 00f322624..a6eb137d2 100644
--- a/zenserver/testing/launch.h
+++ b/zenserver/testing/launch.h
@@ -2,7 +2,7 @@
#pragma once
-#include <zencore/httpserver.h>
+#include <zenhttp/httpserver.h>
#include <spdlog/spdlog.h>
#include <filesystem>
diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp
index 2dd51fe6c..ba6300c65 100644
--- a/zenserver/upstream/jupiter.cpp
+++ b/zenserver/upstream/jupiter.cpp
@@ -81,7 +81,7 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke
cpr::Response Response = Session.Get();
m_Log.debug("GET {}", Response);
-
+
const bool Success = Response.status_code == 200;
const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer();
diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h
index efe9d07ba..9f36704fa 100644
--- a/zenserver/upstream/jupiter.h
+++ b/zenserver/upstream/jupiter.h
@@ -2,9 +2,9 @@
#pragma once
-#include <zencore/httpserver.h>
#include <zencore/refcount.h>
#include <zencore/thread.h>
+#include <zenhttp/httpserver.h>
#include <spdlog/spdlog.h>
diff --git a/zenserver/vfs.cpp b/zenserver/vfs.cpp
index 16b23513f..18d8f1842 100644
--- a/zenserver/vfs.cpp
+++ b/zenserver/vfs.cpp
@@ -2,7 +2,7 @@
#include "vfs.h"
-#if WITH_VFS
+#if ZEN_WITH_VFS
# include <zencore/except.h>
# include <zencore/filesystem.h>
# include <zencore/snapshot_manifest.h>
diff --git a/zenserver/vfs.h b/zenserver/vfs.h
index f8fea6e12..0d2ca6062 100644
--- a/zenserver/vfs.h
+++ b/zenserver/vfs.h
@@ -2,7 +2,11 @@
#pragma once
-#if WITH_VFS
+#ifndef ZEN_WITH_VFS
+# define ZEN_WITH_VFS 0
+#endif
+
+#if ZEN_WITH_VFS
# include <memory>
namespace zen {
diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua
index cbe021f90..bb70846fa 100644
--- a/zenserver/xmake.lua
+++ b/zenserver/xmake.lua
@@ -1,7 +1,7 @@
target("zenserver")
set_kind("binary")
add_files("**.cpp")
- add_deps("zencore", "zenstore", "zenutil")
+ add_deps("zencore", "zenhttp", "zenstore", "zenutil")
add_includedirs(".")
set_symbols("debug")
@@ -23,7 +23,8 @@ target("zenserver")
"vcpkg::sol2",
"vcpkg::lua",
"vcpkg::asio",
- "vcpkg::json11"
+ "vcpkg::json11",
+ "vcpkg::uwebsockets", "vcpkg::usockets", "vcpkg::libuv"
)
add_packages(
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index c857d4c71..c9f74daa4 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -2,7 +2,6 @@
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
-#include <zencore/httpserver.h>
#include <zencore/iobuffer.h>
#include <zencore/refcount.h>
#include <zencore/scopeguard.h>
@@ -10,6 +9,7 @@
#include <zencore/thread.h>
#include <zencore/timer.h>
#include <zencore/windows.h>
+#include <zenhttp/httpserver.h>
#include <zenserverprocess.h>
#include <zenstore/cas.h>
#include <zenstore/cidstore.h>
@@ -72,6 +72,7 @@
#include "diag/diagsvcs.h"
#include "experimental/usnjournal.h"
#include "projectstore.h"
+#include "testing/httptest.h"
#include "testing/launch.h"
#include "upstream/jupiter.h"
#include "upstream/upstreamcache.h"
@@ -205,7 +206,7 @@ public:
else
{
UpstreamCache.reset();
- spdlog::info("upstream cache NOT active");
+ spdlog::info("NOT using upstream cache");
}
}
@@ -221,32 +222,40 @@ public:
{
StartMesh(BasePort);
}
+ else
+ {
+ spdlog::info("NOT starting mesh");
+ }
+
+ m_Http = zen::CreateHttpServer();
+ m_Http->Initialize(BasePort);
+ m_Http->RegisterService(m_HealthService);
+
+ m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics
+ m_Http->RegisterService(m_TestingService);
- m_Http.Initialize(BasePort);
- m_Http.AddEndpoint(m_HealthService);
- m_Http.AddEndpoint(m_TestService);
- m_Http.AddEndpoint(m_AdminService);
+ m_Http->RegisterService(m_AdminService);
if (m_HttpProjectService)
{
- m_Http.AddEndpoint(*m_HttpProjectService);
+ m_Http->RegisterService(*m_HttpProjectService);
}
- m_Http.AddEndpoint(m_CasService);
+ m_Http->RegisterService(m_CasService);
if (m_StructuredCacheService)
{
- m_Http.AddEndpoint(*m_StructuredCacheService);
+ m_Http->RegisterService(*m_StructuredCacheService);
}
if (m_HttpLaunchService)
{
- m_Http.AddEndpoint(*m_HttpLaunchService);
+ m_Http->RegisterService(*m_HttpLaunchService);
}
if (m_HttpFunctionService)
{
- m_Http.AddEndpoint(*m_HttpFunctionService);
+ m_Http->RegisterService(*m_HttpFunctionService);
}
}
@@ -284,7 +293,7 @@ public:
__debugbreak();
}
- m_Http.Run(m_TestMode);
+ m_Http->Run(m_TestMode);
spdlog::info(ZEN_APP_NAME " exiting");
@@ -296,7 +305,7 @@ public:
void RequestExit(int ExitCode)
{
RequestApplicationExit(ExitCode);
- m_Http.RequestExit();
+ m_Http->RequestExit();
}
void Cleanup() { spdlog::info(ZEN_APP_NAME " cleaning up"); }
@@ -358,13 +367,14 @@ private:
zen::ProcessHandle m_Process;
zen::NamedMutex m_ServerMutex;
- zen::HttpServer m_Http;
+ zen::Ref<zen::HttpServer> m_Http;
std::unique_ptr<zen::CasStore> m_CasStore{zen::CreateCasStore()};
std::unique_ptr<zen::CidStore> m_CidStore;
std::unique_ptr<ZenCacheStore> m_CacheStore;
zen::CasGc m_Gc{*m_CasStore};
zen::CasScrubber m_Scrubber{*m_CasStore};
HttpTestService m_TestService;
+ zen::HttpTestingService m_TestingService;
zen::HttpCasService m_CasService{*m_CasStore};
zen::RefPtr<zen::ProjectStore> m_ProjectStore;
zen::Ref<zen::LocalProjectService> m_LocalProjectService;
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index e4b40e13e..aa9d538a5 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -110,6 +110,7 @@
<ClInclude Include="config.h" />
<ClInclude Include="diag\logging.h" />
<ClInclude Include="sos\sos.h" />
+ <ClInclude Include="testing\httptest.h" />
<ClInclude Include="upstream\jupiter.h" />
<ClInclude Include="projectstore.h" />
<ClInclude Include="cache\cacheagent.h" />
@@ -132,6 +133,7 @@
<ClCompile Include="projectstore.cpp" />
<ClCompile Include="cache\cacheagent.cpp" />
<ClCompile Include="sos\sos.cpp" />
+ <ClCompile Include="testing\httptest.cpp" />
<ClCompile Include="upstream\jupiter.cpp" />
<ClCompile Include="testing\launch.cpp" />
<ClCompile Include="cache\cachestore.cpp" />
@@ -146,6 +148,9 @@
<ProjectReference Include="..\zencore\zencore.vcxproj">
<Project>{d75bf9ab-c61e-4fff-ad59-1563430f05e2}</Project>
</ProjectReference>
+ <ProjectReference Include="..\zenhttp\zenhttp.vcxproj">
+ <Project>{8eeb3be5-7001-46bf-aafd-edb7558ac012}</Project>
+ </ProjectReference>
<ProjectReference Include="..\zenstore\zenstore.vcxproj">
<Project>{26cbbaeb-14c1-4efc-877d-80f48215651c}</Project>
</ProjectReference>
@@ -153,6 +158,9 @@
<Project>{77f8315d-b21d-4db0-9a6f-2d3359f88a70}</Project>
</ProjectReference>
</ItemGroup>
+ <ItemGroup>
+ <None Include="xmake.lua" />
+ </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters
index ca16caf77..a86a6d96d 100644
--- a/zenserver/zenserver.vcxproj.filters
+++ b/zenserver/zenserver.vcxproj.filters
@@ -38,6 +38,7 @@
<ClInclude Include="upstream\upstreamcache.h">
<Filter>upstream</Filter>
</ClInclude>
+ <ClInclude Include="testing\httptest.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="zenserver.cpp" />
@@ -71,6 +72,7 @@
<ClCompile Include="upstream\upstreamcache.cpp">
<Filter>upstream</Filter>
</ClCompile>
+ <ClCompile Include="testing\httptest.cpp" />
</ItemGroup>
<ItemGroup>
<Filter Include="cache">
@@ -89,4 +91,7 @@
<UniqueIdentifier>{303c28c2-3607-4ef4-89bd-e3618fe37e74}</UniqueIdentifier>
</Filter>
</ItemGroup>
+ <ItemGroup>
+ <None Include="xmake.lua" />
+ </ItemGroup>
</Project> \ No newline at end of file