diff options
| author | Martin Ridgers <[email protected]> | 2021-09-15 09:22:32 +0200 |
|---|---|---|
| committer | Martin Ridgers <[email protected]> | 2021-09-15 09:23:33 +0200 |
| commit | 8f5e773529858223beeecf5d1b69c23991df644e (patch) | |
| tree | 2c360c67e028f5ecd7368212b0adf8b23578ff9d /zenserver | |
| parent | Use zen::Sleep() in timer.cpp's tests (diff) | |
| parent | Updated function service to new package management API (diff) | |
| download | zen-8f5e773529858223beeecf5d1b69c23991df644e.tar.xz zen-8f5e773529858223beeecf5d1b69c23991df644e.zip | |
Merge main
Diffstat (limited to 'zenserver')
| -rw-r--r-- | zenserver/admin/admin.h | 2 | ||||
| -rw-r--r-- | zenserver/cache/structuredcache.cpp | 63 | ||||
| -rw-r--r-- | zenserver/cache/structuredcache.h | 2 | ||||
| -rw-r--r-- | zenserver/casstore.cpp | 16 | ||||
| -rw-r--r-- | zenserver/casstore.h | 2 | ||||
| -rw-r--r-- | zenserver/compute/apply.cpp | 72 | ||||
| -rw-r--r-- | zenserver/compute/apply.h | 2 | ||||
| -rw-r--r-- | zenserver/diag/diagsvcs.h | 16 | ||||
| -rw-r--r-- | zenserver/diag/logging.cpp | 4 | ||||
| -rw-r--r-- | zenserver/projectstore.cpp | 123 | ||||
| -rw-r--r-- | zenserver/projectstore.h | 2 | ||||
| -rw-r--r-- | zenserver/sos/sos.h | 2 | ||||
| -rw-r--r-- | zenserver/testing/httptest.cpp | 106 | ||||
| -rw-r--r-- | zenserver/testing/httptest.h | 47 | ||||
| -rw-r--r-- | zenserver/testing/launch.cpp | 6 | ||||
| -rw-r--r-- | zenserver/testing/launch.h | 2 | ||||
| -rw-r--r-- | zenserver/upstream/jupiter.cpp | 2 | ||||
| -rw-r--r-- | zenserver/upstream/jupiter.h | 2 | ||||
| -rw-r--r-- | zenserver/vfs.cpp | 2 | ||||
| -rw-r--r-- | zenserver/vfs.h | 6 | ||||
| -rw-r--r-- | zenserver/xmake.lua | 5 | ||||
| -rw-r--r-- | zenserver/zenserver.cpp | 38 | ||||
| -rw-r--r-- | zenserver/zenserver.vcxproj | 8 | ||||
| -rw-r--r-- | zenserver/zenserver.vcxproj.filters | 5 |
24 files changed, 366 insertions, 169 deletions
diff --git a/zenserver/admin/admin.h b/zenserver/admin/admin.h index 3bb8a9158..f90ad4537 100644 --- a/zenserver/admin/admin.h +++ b/zenserver/admin/admin.h @@ -2,7 +2,7 @@ #pragma once -#include <zencore/httpserver.h> +#include <zenhttp/httpserver.h> class HttpAdminService : public zen::HttpService { diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 0e235a9be..b2f8d191c 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -4,8 +4,8 @@ #include <zencore/compactbinaryvalidation.h> #include <zencore/compress.h> #include <zencore/fmtutils.h> -#include <zencore/httpserver.h> #include <zencore/timer.h> +#include <zenhttp/httpserver.h> #include "structuredcache.h" #include "structuredcachestore.h" @@ -25,27 +25,6 @@ namespace zen { using namespace std::literals; -zen::HttpContentType -MapToHttpContentType(zen::ZenContentType Type) -{ - switch (Type) - { - default: - case zen::ZenContentType::kBinary: - return zen::HttpContentType::kBinary; - case zen::ZenContentType::kCbObject: - return zen::HttpContentType::kCbObject; - case zen::ZenContentType::kCbPackage: - return zen::HttpContentType::kCbPackage; - case zen::ZenContentType::kText: - return zen::HttpContentType::kText; - case zen::ZenContentType::kJSON: - return zen::HttpContentType::kJSON; - case zen::ZenContentType::kYAML: - return zen::HttpContentType::kYAML; - } -}; - ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(::ZenCacheStore& InCacheStore, @@ -93,7 +72,7 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) return HandleCacheBucketRequest(Request, Key); } - return Request.WriteResponse(zen::HttpResponse::BadRequest); // invalid URL + return Request.WriteResponse(zen::HttpResponseCode::BadRequest); // invalid URL } if (Ref.PayloadId == IoHash::Zero) @@ -128,11 +107,11 @@ HttpStructuredCacheService::HandleCacheBucketRequest(zen::HttpServerRequest& Req if (m_CacheStore.DropBucket(Bucket)) { - return Request.WriteResponse(zen::HttpResponse::OK); + return Request.WriteResponse(zen::HttpResponseCode::OK); } else { - return Request.WriteResponse(zen::HttpResponse::NotFound); + return Request.WriteResponse(zen::HttpResponseCode::NotFound); } break; } @@ -209,7 +188,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { m_Log.debug("MISS - '{}/{}'", Ref.BucketSegment, Ref.HashKey); - return Request.WriteResponse(zen::HttpResponse::NotFound); + return Request.WriteResponse(zen::HttpResponseCode::NotFound); } if (Verb == kHead) @@ -224,7 +203,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req Value.Value.GetContentType(), InUpstreamCache ? "upstream" : "local"); - return Request.WriteResponse(zen::HttpResponse::OK, MapToHttpContentType(Value.Value.GetContentType()), Value.Value); + return Request.WriteResponse(zen::HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); } break; @@ -234,7 +213,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req if (!Body || Body.Size() == 0) { - return Request.WriteResponse(zen::HttpResponse::BadRequest); + return Request.WriteResponse(zen::HttpResponseCode::BadRequest); } const HttpContentType ContentType = Request.RequestContentType(); @@ -253,7 +232,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req break; default: - return Request.WriteResponse(zen::HttpResponse::BadRequest); + return Request.WriteResponse(zen::HttpResponseCode::BadRequest); } if (!IsCompactBinary) @@ -272,7 +251,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req {.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}}); } - return Request.WriteResponse(zen::HttpResponse::Created); + return Request.WriteResponse(zen::HttpResponseCode::Created); } // Validate payload before accessing it @@ -284,7 +263,9 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req m_Log.warn("Payload for key '{}/{}' ({} bytes) failed validation", Ref.BucketSegment, Ref.HashKey, Body.Size()); // TODO: add details in response, kText || kCbObject? - return Request.WriteResponse(HttpResponse::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); + return Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary validation failed"sv); } // Extract referenced payload hashes @@ -336,7 +317,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req .PayloadIds = std::move(References)}); } - return Request.WriteResponse(zen::HttpResponse::Created); + return Request.WriteResponse(zen::HttpResponseCode::Created); } else { @@ -351,7 +332,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req Response.EndArray(); // Return Created | BadRequest? - return Request.WriteResponse(zen::HttpResponse::Created, Response.Save()); + return Request.WriteResponse(zen::HttpResponseCode::Created, Response.Save()); } } break; @@ -407,7 +388,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (!Payload) { m_Log.debug("MISS - '{}/{}/{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId); - return Request.WriteResponse(zen::HttpResponse::NotFound); + return Request.WriteResponse(zen::HttpResponseCode::NotFound); } m_Log.debug("HIT - '{}/{}/{}' ({} bytes, {}) ({})", @@ -423,7 +404,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re Request.SetSuppressResponseBody(); } - return Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Payload); + return Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Payload); } break; @@ -433,7 +414,9 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re { if (Body.Size() == 0) { - return Request.WriteResponse(zen::HttpResponse::BadRequest, HttpContentType::kText, "Empty payload not permitted"); + return Request.WriteResponse(zen::HttpResponseCode::BadRequest, + HttpContentType::kText, + "Empty payload not permitted"); } zen::IoHash ChunkHash = zen::IoHash::HashBuffer(Body); @@ -443,7 +426,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (!Compressed) { // All attachment payloads need to be in compressed buffer format - return Request.WriteResponse(zen::HttpResponse::BadRequest, + return Request.WriteResponse(zen::HttpResponseCode::BadRequest, HttpContentType::kText, "Attachments must be compressed"); } @@ -452,7 +435,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Ref.PayloadId) { // the URL specified content id and content hashes don't match! - return Request.WriteResponse(HttpResponse::BadRequest); + return Request.WriteResponse(HttpResponseCode::BadRequest); } zen::CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash); @@ -469,11 +452,11 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (Result.New) { - return Request.WriteResponse(zen::HttpResponse::Created); + return Request.WriteResponse(zen::HttpResponseCode::Created); } else { - return Request.WriteResponse(zen::HttpResponse::OK); + return Request.WriteResponse(zen::HttpResponseCode::OK); } } } diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index b90301d84..d4bb94c52 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -2,7 +2,7 @@ #pragma once -#include <zencore/httpserver.h> +#include <zenhttp/httpserver.h> #include <spdlog/spdlog.h> #include <memory> diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp index 1d147024a..6f1e4873b 100644 --- a/zenserver/casstore.cpp +++ b/zenserver/casstore.cpp @@ -23,7 +23,7 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) if ((EntryCount * sizeof(IoHash)) != Payload.Size()) { - return ServerRequest.WriteResponse(HttpResponse::BadRequest); + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest); } const IoHash* Hashes = reinterpret_cast<const IoHash*>(Payload.Data()); @@ -55,7 +55,7 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) Values[0] = IoBufferBuilder::MakeCloneFromMemory(HeaderStream.Data(), HeaderStream.Size()); - ServerRequest.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Values); + ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Values); }, HttpVerb::kPost); @@ -74,10 +74,10 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) { if (IoBuffer Value = m_CasStore.FindChunk(Hash)) { - return ServerRequest.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Value); + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value); } - return ServerRequest.WriteResponse(HttpResponse::NotFound); + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); } break; @@ -89,12 +89,12 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) // URI hash must match content hash if (PayloadHash != Hash) { - return ServerRequest.WriteResponse(HttpResponse::BadRequest); + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest); } m_CasStore.InsertChunk(Payload, PayloadHash); - return ServerRequest.WriteResponse(HttpResponse::OK); + return ServerRequest.WriteResponse(HttpResponseCode::OK); } break; } @@ -129,11 +129,11 @@ HttpCasService::HandleRequest(zen::HttpServerRequest& Request) if (InsertResult.New) { - return Request.WriteResponse(HttpResponse::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else { - return Request.WriteResponse(HttpResponse::OK); + return Request.WriteResponse(HttpResponseCode::OK); } } break; diff --git a/zenserver/casstore.h b/zenserver/casstore.h index 7166f796e..4ca6908b5 100644 --- a/zenserver/casstore.h +++ b/zenserver/casstore.h @@ -2,7 +2,7 @@ #pragma once -#include <zencore/httpserver.h> +#include <zenhttp/httpserver.h> #include <zenstore/cas.h> namespace zen { diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 939ac3362..94dedf087 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -351,12 +351,12 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, if (auto It = m_WorkerMap.find(WorkerId); It == m_WorkerMap.end()) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } else { const WorkerDesc& Desc = It->second; - return HttpReq.WriteResponse(HttpResponse::OK, Desc.Descriptor); + return HttpReq.WriteResponse(HttpResponseCode::OK, Desc.Descriptor); } } break; @@ -378,6 +378,10 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, ChunkSet.AddChunk(Hash); }); + // Note that we store executables uncompressed to make it + // more straightforward and efficient to materialize them, hence + // the CAS lookup here instead of CID for the input payloads + m_CasStore.FilterChunks(ChunkSet); if (ChunkSet.IsEmpty()) @@ -388,7 +392,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, spdlog::debug("worker {}: all attachments already available", WorkerId); - return HttpReq.WriteResponse(HttpResponse::NoContent); + return HttpReq.WriteResponse(HttpResponseCode::NoContent); } else { @@ -406,7 +410,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, spdlog::debug("worker {}: need {} attachments", WorkerId, ChunkSet.GetChunkSet().size()); - return HttpReq.WriteResponse(HttpResponse::NotFound, ResponseWriter.Save()); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, ResponseWriter.Save()); } } break; @@ -426,21 +430,25 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, for (const CbAttachment& Attachment : Attachments) { - ZEN_ASSERT(Attachment.IsBinary()); + ZEN_ASSERT(Attachment.IsCompressedBinary()); - const IoHash DataHash = Attachment.GetHash(); - CompressedBuffer DataView = Attachment.AsCompressedBinary(); + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); + SharedBuffer Decompressed = DataView.Decompress(); + const uint64_t DecompressedSize = DataView.GetRawSize(); - TotalAttachmentBytes += DataView.GetCompressedSize(); + TotalAttachmentBytes += DecompressedSize; ++AttachmentCount; - IoBuffer Payload = DataView.GetCompressed().Flatten().AsIoBuffer(); + // Note that we store executables uncompressed to make it + // more straightforward and efficient to materialize them - CasStore::InsertResult InsertResult = m_CasStore.InsertChunk(Payload, DataHash); + const CasStore::InsertResult InsertResult = + m_CasStore.InsertChunk(Decompressed.AsIoBuffer(), IoHash::FromBLAKE3(DataView.GetRawHash())); if (InsertResult.New) { - TotalNewBytes += Payload.Size(); + TotalNewBytes += DecompressedSize; ++NewAttachmentCount; } } @@ -456,7 +464,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, m_WorkerMap.insert_or_assign(WorkerId, WorkerDesc{.Descriptor = Obj}); - return HttpReq.WriteResponse(HttpResponse::NoContent); + return HttpReq.WriteResponse(HttpResponseCode::NoContent); } break; } @@ -495,7 +503,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, if (auto It = m_WorkerMap.find(WorkerId); It == m_WorkerMap.end()) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } else { @@ -526,7 +534,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, RequestObject.IterateAttachments([&](CbFieldView Field) { const IoHash FileHash = Field.AsHash(); - if (!m_CasStore.FindChunk(FileHash)) + if (!m_CidStore.ContainsChunk(FileHash)) { NeedList.push_back(FileHash); } @@ -538,7 +546,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, CbPackage Output = ExecAction(Worker, RequestObject); - return HttpReq.WriteResponse(HttpResponse::OK, Output); + return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } CbObjectWriter Cbo; @@ -552,7 +560,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, Cbo.EndArray(); CbObject Response = Cbo.Save(); - return HttpReq.WriteResponse(HttpResponse::NotFound, Response); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Response); } break; @@ -570,19 +578,21 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, for (const CbAttachment& Attachment : Attachments) { - ZEN_ASSERT(Attachment.IsBinary()); + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); - const IoHash DataHash = Attachment.GetHash(); - SharedBuffer DataView = Attachment.AsBinary(); + const uint64_t CompressedSize = DataView.GetCompressedSize(); - TotalAttachmentBytes += DataView.GetSize(); + TotalAttachmentBytes += CompressedSize; ++AttachmentCount; - CasStore::InsertResult InsertResult = m_CasStore.InsertChunk(DataView.AsIoBuffer(), DataHash); + const CasStore::InsertResult InsertResult = m_CidStore.AddChunk(DataView); if (InsertResult.New) { - TotalNewBytes += DataView.GetSize(); + TotalNewBytes += CompressedSize; ++NewAttachmentCount; } } @@ -595,7 +605,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, CbPackage Output = ExecAction(Worker, ActionObj); - return HttpReq.WriteResponse(HttpResponse::OK, Output); + return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } break; } @@ -659,7 +669,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, if (!AllOk) { // TODO: Could report all the missing pieces in the response here - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } std::string Executable8{RequestObject["cmd"].AsString()}; @@ -681,7 +691,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, Response << "exitcode" << Job.ExitCode(); - return HttpReq.WriteResponse(HttpResponse::OK, Response.Save()); + return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); } break; } @@ -780,18 +790,16 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) // Manifest inputs in sandbox Action.IterateAttachments([&](CbFieldView Field) { - const IoHash Hash = Field.AsHash(); - std::filesystem::path FilePath{SandboxPath / "Inputs" / Hash.ToHexString()}; - IoBuffer DataBuffer = m_CasStore.FindChunk(Hash); + const IoHash Cid = Field.AsHash(); + std::filesystem::path FilePath{SandboxPath / "Inputs" / Cid.ToHexString()}; + IoBuffer DataBuffer = m_CidStore.FindChunkByCid(Cid); if (!DataBuffer) { throw std::exception("Chunk missing" /* ADD CONTEXT */); } - CompressedBuffer Buffer = CompressedBuffer::Compress(SharedBuffer(std::move(DataBuffer))); - - zen::WriteFile(FilePath, Buffer.GetCompressed().Flatten().AsIoBuffer()); + zen::WriteFile(FilePath, DataBuffer); }); // Set up environment variables @@ -884,7 +892,7 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) ZEN_ASSERT(OutputData.Data.size() == 1); - CbAttachment Attachment(SharedBuffer(ChunkData.Data[0]), Hash); + CbAttachment Attachment(CompressedBuffer::FromCompressed(SharedBuffer(ChunkData.Data[0]))); OutputPackage.AddAttachment(Attachment); }); diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index 20a58507b..695dc2e6e 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -3,9 +3,9 @@ #pragma once #include <zencore/compactbinary.h> -#include <zencore/httpserver.h> #include <zencore/iohash.h> #include <zencore/logging.h> +#include <zenhttp/httpserver.h> #include <filesystem> #include <unordered_map> diff --git a/zenserver/diag/diagsvcs.h b/zenserver/diag/diagsvcs.h index 84f8d22ee..51ee98f67 100644 --- a/zenserver/diag/diagsvcs.h +++ b/zenserver/diag/diagsvcs.h @@ -2,8 +2,8 @@ #pragma once -#include <zencore/httpserver.h> #include <zencore/iobuffer.h> +#include <zenhttp/httpserver.h> ////////////////////////////////////////////////////////////////////////// @@ -25,17 +25,17 @@ public: if (Uri == "hello"sv) { - Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kText, u8"hello world!"sv); + Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kText, u8"hello world!"sv); // OutputLogMessageInternal(&LogPoint, 0, 0); } else if (Uri == "1K"sv) { - Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, m_1k); + Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, m_1k); } else if (Uri == "1M"sv) { - Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, m_1m); + Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, m_1m); } else if (Uri == "1M_1k"sv) { @@ -47,7 +47,7 @@ public: Buffers.push_back(m_1k); } - Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers); + Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Buffers); } else if (Uri == "1G"sv) { @@ -59,7 +59,7 @@ public: Buffers.push_back(m_1m); } - Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers); + Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Buffers); } else if (Uri == "1G_1k"sv) { @@ -71,7 +71,7 @@ public: Buffers.push_back(m_1k); } - Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers); + Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Buffers); } } @@ -95,7 +95,7 @@ public: switch (Request.RequestVerb()) { case zen::HttpVerb::kGet: - return Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kText, u8"OK!"sv); + return Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kText, u8"OK!"sv); } } diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 796a15d01..5782ce582 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -195,13 +195,13 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) std::filesystem::path LogPath = GlobalOptions.DataDir / "logs/zenserver.log"; - bool IsAsync = true; + bool IsAsync = true; spdlog::level::level_enum LogLevel = spdlog::level::info; if (GlobalOptions.IsDebug) { LogLevel = spdlog::level::debug; - IsAsync = false; + IsAsync = false; } if (IsAsync) diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 006796b28..2bbc1dce3 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -542,8 +542,30 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage) for (const auto& Attach : Attachments) { - IoBuffer AttachmentData = Attach.AsBinary().AsIoBuffer(); - CasStore::InsertResult Result = m_CasStore.InsertChunk(AttachmentData, Attach.GetHash()); + IoBuffer AttachmentData; + + if (Attach.IsBinary()) + { + AttachmentData = Attach.AsBinary().AsIoBuffer(); + } + else if (Attach.IsCompressedBinary()) + { + ZEN_NOT_IMPLEMENTED("Compressed binary attachments are currently not supported for oplogs"); + + AttachmentData = Attach.AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer(); + } + else if (Attach.IsObject()) + { + AttachmentData = Attach.AsObject().GetBuffer().AsIoBuffer(); + } + else + { + ZEN_NOT_IMPLEMENTED("Unknown attachment type"); + } + + ZEN_ASSERT(AttachmentData); + + CasStore::InsertResult Result = m_CasStore.InsertChunk(AttachmentData, Attach.GetHash()); const uint64_t AttachmentSize = AttachmentData.Size(); @@ -909,7 +931,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (FoundLog == nullptr) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } // Parse Request @@ -940,7 +962,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (Payload.Size() <= sizeof(RequestHeader)) { - HttpReq.WriteResponse(HttpResponse::BadRequest); + HttpReq.WriteResponse(HttpResponseCode::BadRequest); } RequestHeader RequestHdr; @@ -948,7 +970,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (RequestHdr.Magic != RequestHeader::kMagic) { - HttpReq.WriteResponse(HttpResponse::BadRequest); + HttpReq.WriteResponse(HttpResponseCode::BadRequest); } std::vector<RequestChunkEntry> RequestedChunks; @@ -1004,9 +1026,9 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) ResponsePtr += sizeof(ResponseHdr); for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) { - //const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; - const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]); - ResponseChunkEntry ResponseChunk; + // const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; + const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]); + ResponseChunkEntry ResponseChunk; ResponseChunk.CorrelationId = ChunkIndex; if (FoundChunk) { @@ -1020,7 +1042,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) ResponsePtr += sizeof(ResponseChunk); } - return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, OutBlobs); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs); }, HttpVerb::kPost); @@ -1038,7 +1060,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (FoundLog == nullptr) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } ProjectStore::Oplog& Log = *FoundLog; @@ -1063,7 +1085,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) Response.EndArray(); - return HttpReq.WriteResponse(HttpResponse::OK, Response.Save()); + return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kGet); @@ -1080,7 +1102,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (FoundLog == nullptr) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } ProjectStore::Oplog& Log = *FoundLog; @@ -1093,10 +1115,10 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) { CbObjectWriter Response; Response << "size" << Value.Size(); - return HttpReq.WriteResponse(HttpResponse::OK, Response.Save()); + return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); } - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); }, HttpVerb::kGet); @@ -1124,7 +1146,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } else { - return HttpReq.WriteResponse(HttpResponse::BadRequest); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); } } @@ -1137,7 +1159,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } else { - return HttpReq.WriteResponse(HttpResponse::BadRequest); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); } } @@ -1147,7 +1169,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (FoundLog == nullptr) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } ProjectStore::Oplog& Log = *FoundLog; @@ -1162,7 +1184,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) case HttpVerb::kGet: if (!Value) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } if (Verb == HttpVerb::kHead) @@ -1185,10 +1207,10 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) // Send only a subset of data IoBuffer InnerValue(Value, Offset, Size); - return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, InnerValue); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, InnerValue); } - return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Value); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value); } }, HttpVerb::kGet | HttpVerb::kHead); @@ -1217,7 +1239,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } else { - return HttpReq.WriteResponse(HttpResponse::BadRequest); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); } } @@ -1230,7 +1252,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } else { - return HttpReq.WriteResponse(HttpResponse::BadRequest); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); } } @@ -1240,7 +1262,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) IoBuffer Value = m_CasStore.FindChunk(Hash); if (!Value) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } if (IsOffset) @@ -1258,10 +1280,10 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) // Send only a subset of data IoBuffer InnerValue(Value, Offset, Size); - return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, InnerValue); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, InnerValue); } - return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Value); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value); }, HttpVerb::kGet); @@ -1277,7 +1299,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (FoundLog == nullptr) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } // This operation takes a list of referenced hashes and decides which @@ -1312,7 +1334,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) Cbo.EndArray(); CbObject Response = Cbo.Save(); - return HttpReq.WriteResponse(HttpResponse::OK, Response); + return HttpReq.WriteResponse(HttpResponseCode::OK, Response); }, HttpVerb::kPost); @@ -1340,7 +1362,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (FoundLog == nullptr) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } ProjectStore::Oplog& Log = *FoundLog; @@ -1387,23 +1409,26 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) }; CbPackage Package; - if (!Package.TryLoad(Payload, &UniqueBuffer::Alloc, &Resolver)) + + if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) { - return HttpReq.WriteResponse(HttpResponse::BadRequest, HttpContentType::kText, "Invalid package"); + m_Log.error("Received malformed package!"); + + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"); } if (!IsValid) { // TODO: emit diagnostics identifying missing chunks - return HttpReq.WriteResponse(HttpResponse::NotFound, HttpContentType::kText, "Missing chunk reference"); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing chunk reference"); } CbObject Core = Package.GetObject(); if (!Core["key"sv]) { - return HttpReq.WriteResponse(HttpResponse::BadRequest, HttpContentType::kText, "No oplog entry key specified"); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified"); } // Write core to oplog @@ -1412,12 +1437,12 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (OpLsn == ProjectStore::Oplog::kInvalidOp) { - return HttpReq.WriteResponse(HttpResponse::BadRequest); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); } m_Log.info("new op #{:4} - {}/{} ({:>6}) {}", OpLsn, ProjectId, OplogId, NiceBytes(Payload.Size()), Core["key"sv].AsString()); - HttpReq.WriteResponse(HttpResponse::Created); + HttpReq.WriteResponse(HttpResponseCode::Created); }, HttpVerb::kPost); @@ -1428,7 +1453,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) // TODO: look up op and respond with the payload! - HttpReq.WriteResponse(HttpResponse::Accepted, HttpContentType::kText, u8"yeee"sv); + HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, u8"yeee"sv); }, HttpVerb::kGet); @@ -1444,7 +1469,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (!ProjectIt) { - return Req.ServerRequest().WriteResponse(HttpResponse::NotFound, + return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "project {} not found"_format(ProjectId)); } @@ -1459,7 +1484,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (!OplogIt) { - return Req.ServerRequest().WriteResponse(HttpResponse::NotFound, + return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "oplog {} not found in project {}"_format(OplogId, ProjectId)); } @@ -1469,7 +1494,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) CbObjectWriter Cb; Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempDir(); - Req.ServerRequest().WriteResponse(HttpResponse::OK, Cb.Save()); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save()); } break; @@ -1482,18 +1507,18 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (!Prj.NewOplog(OplogId)) { // TODO: indicate why the operation failed! - return Req.ServerRequest().WriteResponse(HttpResponse::InternalServerError); + return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError); } m_Log.info("established oplog {} / {}", ProjectId, OplogId); - return Req.ServerRequest().WriteResponse(HttpResponse::Created); + return Req.ServerRequest().WriteResponse(HttpResponseCode::Created); } // I guess this should ultimately be used to execute RPCs but for now, it // does absolutely nothing - return Req.ServerRequest().WriteResponse(HttpResponse::BadRequest); + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } break; @@ -1503,7 +1528,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) ProjectIt->DeleteOplog(OplogId); - return Req.ServerRequest().WriteResponse(HttpResponse::OK); + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK); } break; } @@ -1522,7 +1547,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (FoundLog == nullptr) { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } CbObjectWriter Response; @@ -1542,7 +1567,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } else { - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } } else @@ -1555,7 +1580,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } } - return HttpReq.WriteResponse(HttpResponse::OK, Response.Save()); + return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kGet); @@ -1585,7 +1610,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) EngineRoot, ProjectRoot); - Req.ServerRequest().WriteResponse(HttpResponse::Created); + Req.ServerRequest().WriteResponse(HttpResponseCode::Created); } break; @@ -1595,7 +1620,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (!ProjectIt) { - return Req.ServerRequest().WriteResponse(HttpResponse::NotFound, + return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "project {} not found"_format(ProjectId)); } @@ -1609,7 +1634,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) Prj.IterateOplogs([&](const ProjectStore::Oplog& I) { Response << "id"sv << I.OplogId(); }); Response.EndArray(); // oplogs - Req.ServerRequest().WriteResponse(HttpResponse::OK, Response.Save()); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); } break; @@ -1619,7 +1644,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (!ProjectIt) { - return Req.ServerRequest().WriteResponse(HttpResponse::NotFound, + return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "project {} not found"_format(ProjectId)); } diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h index 33bacfdaa..8fe189ab9 100644 --- a/zenserver/projectstore.h +++ b/zenserver/projectstore.h @@ -2,9 +2,9 @@ #pragma once -#include <zencore/httpserver.h> #include <zencore/uid.h> #include <zencore/xxhash.h> +#include <zenhttp/httpserver.h> #include <zenstore/cas.h> #include <zenstore/caslog.h> diff --git a/zenserver/sos/sos.h b/zenserver/sos/sos.h index 283735cbd..da9064262 100644 --- a/zenserver/sos/sos.h +++ b/zenserver/sos/sos.h @@ -2,7 +2,7 @@ #pragma once -#include <zencore/httpserver.h> +#include <zenhttp/httpserver.h> #include <spdlog/spdlog.h> diff --git a/zenserver/testing/httptest.cpp b/zenserver/testing/httptest.cpp new file mode 100644 index 000000000..c4fd6003c --- /dev/null +++ b/zenserver/testing/httptest.cpp @@ -0,0 +1,106 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httptest.h" + +#include <zencore/compactbinarypackage.h> + +namespace zen { + +HttpTestingService::HttpTestingService() +{ + m_Router.RegisterRoute( + "hello", + [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "echo", + [this](HttpRouterRequest& Req) { + IoBuffer Body = Req.ServerRequest().ReadPayload(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Body); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "package", + [this](HttpRouterRequest& Req) { + CbPackage Pkg = Req.ServerRequest().ReadPayloadPackage(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Pkg); + }, + HttpVerb::kPost); +} + +HttpTestingService::~HttpTestingService() +{ +} + +const char* +HttpTestingService::BaseUri() const +{ + return "/testing/"; +} + +void +HttpTestingService::HandleRequest(HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +Ref<IHttpPackageHandler> +HttpTestingService::HandlePackageRequest(HttpServerRequest& HttpServiceRequest) +{ + RwLock::ExclusiveLockScope _(m_RwLock); + + const uint32_t RequestId = HttpServiceRequest.RequestId(); + + if (auto It = m_HandlerMap.find(RequestId); It != m_HandlerMap.end()) + { + Ref<HttpTestingService::PackageHandler> Handler = std::move(It->second); + + m_HandlerMap.erase(It); + + return Handler.Get(); + } + + auto InsertResult = m_HandlerMap.insert({RequestId, nullptr}); + + _.ReleaseNow(); + + return (InsertResult.first->second = new PackageHandler(*this, RequestId)).Get(); +} + +////////////////////////////////////////////////////////////////////////// + +HttpTestingService::PackageHandler::PackageHandler(HttpTestingService& Svc, uint32_t RequestId) : m_Svc(Svc), m_RequestId(RequestId) +{ +} + +HttpTestingService::PackageHandler::~PackageHandler() +{ +} + +void +HttpTestingService::PackageHandler::FilterOffer(std::vector<IoHash>& OfferCids) +{ + ZEN_UNUSED(OfferCids); + // No-op + return; +} +void +HttpTestingService::PackageHandler::OnRequestBegin() +{ +} + +void +HttpTestingService::PackageHandler::OnRequestComplete() +{ +} + +IoBuffer +HttpTestingService::PackageHandler::CreateTarget(const IoHash& Cid, uint64_t StorageSize) +{ + ZEN_UNUSED(Cid); + return IoBuffer{StorageSize}; +} + +} // namespace zen diff --git a/zenserver/testing/httptest.h b/zenserver/testing/httptest.h new file mode 100644 index 000000000..5809d4e2e --- /dev/null +++ b/zenserver/testing/httptest.h @@ -0,0 +1,47 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenhttp/httpserver.h> + +#include <spdlog/spdlog.h> + +namespace zen { + +/** + * Test service to facilitate testing the HTTP framework and client interactions + */ +class HttpTestingService : public HttpService +{ +public: + HttpTestingService(); + ~HttpTestingService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + virtual Ref<IHttpPackageHandler> HandlePackageRequest(HttpServerRequest& HttpServiceRequest) override; + + class PackageHandler : public IHttpPackageHandler + { + public: + PackageHandler(HttpTestingService& Svc, uint32_t RequestId); + ~PackageHandler(); + + virtual void FilterOffer(std::vector<IoHash>& OfferCids) override; + virtual void OnRequestBegin() override; + virtual IoBuffer CreateTarget(const IoHash& Cid, uint64_t StorageSize) override; + virtual void OnRequestComplete() override; + + private: + HttpTestingService& m_Svc; + uint32_t m_RequestId; + }; + +private: + HttpRequestRouter m_Router; + + RwLock m_RwLock; + std::unordered_map<uint32_t, Ref<PackageHandler>> m_HandlerMap; +}; + +} // namespace zen diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp index d06fae3e2..b031193d5 100644 --- a/zenserver/testing/launch.cpp +++ b/zenserver/testing/launch.cpp @@ -409,7 +409,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat Cbo.EndArray(); CbObject Response = Cbo.Save(); - return HttpReq.WriteResponse(HttpResponse::OK, Response); + return HttpReq.WriteResponse(HttpResponseCode::OK, Response); } break; } @@ -470,7 +470,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat if (!AllOk) { // TODO: Could report all the missing pieces in the response here - return HttpReq.WriteResponse(HttpResponse::NotFound); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } std::string Executable8{RequestObject["cmd"].AsString()}; @@ -492,7 +492,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat Response << "exitcode" << Job.ExitCode(); - return HttpReq.WriteResponse(HttpResponse::OK, Response.Save()); + return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); } break; } diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h index 00f322624..a6eb137d2 100644 --- a/zenserver/testing/launch.h +++ b/zenserver/testing/launch.h @@ -2,7 +2,7 @@ #pragma once -#include <zencore/httpserver.h> +#include <zenhttp/httpserver.h> #include <spdlog/spdlog.h> #include <filesystem> diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 2dd51fe6c..ba6300c65 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -81,7 +81,7 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke cpr::Response Response = Session.Get(); m_Log.debug("GET {}", Response); - + const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index efe9d07ba..9f36704fa 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -2,9 +2,9 @@ #pragma once -#include <zencore/httpserver.h> #include <zencore/refcount.h> #include <zencore/thread.h> +#include <zenhttp/httpserver.h> #include <spdlog/spdlog.h> diff --git a/zenserver/vfs.cpp b/zenserver/vfs.cpp index 16b23513f..18d8f1842 100644 --- a/zenserver/vfs.cpp +++ b/zenserver/vfs.cpp @@ -2,7 +2,7 @@ #include "vfs.h" -#if WITH_VFS +#if ZEN_WITH_VFS # include <zencore/except.h> # include <zencore/filesystem.h> # include <zencore/snapshot_manifest.h> diff --git a/zenserver/vfs.h b/zenserver/vfs.h index f8fea6e12..0d2ca6062 100644 --- a/zenserver/vfs.h +++ b/zenserver/vfs.h @@ -2,7 +2,11 @@ #pragma once -#if WITH_VFS +#ifndef ZEN_WITH_VFS +# define ZEN_WITH_VFS 0 +#endif + +#if ZEN_WITH_VFS # include <memory> namespace zen { diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index cbe021f90..bb70846fa 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -1,7 +1,7 @@ target("zenserver") set_kind("binary") add_files("**.cpp") - add_deps("zencore", "zenstore", "zenutil") + add_deps("zencore", "zenhttp", "zenstore", "zenutil") add_includedirs(".") set_symbols("debug") @@ -23,7 +23,8 @@ target("zenserver") "vcpkg::sol2", "vcpkg::lua", "vcpkg::asio", - "vcpkg::json11" + "vcpkg::json11", + "vcpkg::uwebsockets", "vcpkg::usockets", "vcpkg::libuv" ) add_packages( diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index c857d4c71..c9f74daa4 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -2,7 +2,6 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> -#include <zencore/httpserver.h> #include <zencore/iobuffer.h> #include <zencore/refcount.h> #include <zencore/scopeguard.h> @@ -10,6 +9,7 @@ #include <zencore/thread.h> #include <zencore/timer.h> #include <zencore/windows.h> +#include <zenhttp/httpserver.h> #include <zenserverprocess.h> #include <zenstore/cas.h> #include <zenstore/cidstore.h> @@ -72,6 +72,7 @@ #include "diag/diagsvcs.h" #include "experimental/usnjournal.h" #include "projectstore.h" +#include "testing/httptest.h" #include "testing/launch.h" #include "upstream/jupiter.h" #include "upstream/upstreamcache.h" @@ -205,7 +206,7 @@ public: else { UpstreamCache.reset(); - spdlog::info("upstream cache NOT active"); + spdlog::info("NOT using upstream cache"); } } @@ -221,32 +222,40 @@ public: { StartMesh(BasePort); } + else + { + spdlog::info("NOT starting mesh"); + } + + m_Http = zen::CreateHttpServer(); + m_Http->Initialize(BasePort); + m_Http->RegisterService(m_HealthService); + + m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics + m_Http->RegisterService(m_TestingService); - m_Http.Initialize(BasePort); - m_Http.AddEndpoint(m_HealthService); - m_Http.AddEndpoint(m_TestService); - m_Http.AddEndpoint(m_AdminService); + m_Http->RegisterService(m_AdminService); if (m_HttpProjectService) { - m_Http.AddEndpoint(*m_HttpProjectService); + m_Http->RegisterService(*m_HttpProjectService); } - m_Http.AddEndpoint(m_CasService); + m_Http->RegisterService(m_CasService); if (m_StructuredCacheService) { - m_Http.AddEndpoint(*m_StructuredCacheService); + m_Http->RegisterService(*m_StructuredCacheService); } if (m_HttpLaunchService) { - m_Http.AddEndpoint(*m_HttpLaunchService); + m_Http->RegisterService(*m_HttpLaunchService); } if (m_HttpFunctionService) { - m_Http.AddEndpoint(*m_HttpFunctionService); + m_Http->RegisterService(*m_HttpFunctionService); } } @@ -284,7 +293,7 @@ public: __debugbreak(); } - m_Http.Run(m_TestMode); + m_Http->Run(m_TestMode); spdlog::info(ZEN_APP_NAME " exiting"); @@ -296,7 +305,7 @@ public: void RequestExit(int ExitCode) { RequestApplicationExit(ExitCode); - m_Http.RequestExit(); + m_Http->RequestExit(); } void Cleanup() { spdlog::info(ZEN_APP_NAME " cleaning up"); } @@ -358,13 +367,14 @@ private: zen::ProcessHandle m_Process; zen::NamedMutex m_ServerMutex; - zen::HttpServer m_Http; + zen::Ref<zen::HttpServer> m_Http; std::unique_ptr<zen::CasStore> m_CasStore{zen::CreateCasStore()}; std::unique_ptr<zen::CidStore> m_CidStore; std::unique_ptr<ZenCacheStore> m_CacheStore; zen::CasGc m_Gc{*m_CasStore}; zen::CasScrubber m_Scrubber{*m_CasStore}; HttpTestService m_TestService; + zen::HttpTestingService m_TestingService; zen::HttpCasService m_CasService{*m_CasStore}; zen::RefPtr<zen::ProjectStore> m_ProjectStore; zen::Ref<zen::LocalProjectService> m_LocalProjectService; diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index e4b40e13e..aa9d538a5 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -110,6 +110,7 @@ <ClInclude Include="config.h" /> <ClInclude Include="diag\logging.h" /> <ClInclude Include="sos\sos.h" /> + <ClInclude Include="testing\httptest.h" /> <ClInclude Include="upstream\jupiter.h" /> <ClInclude Include="projectstore.h" /> <ClInclude Include="cache\cacheagent.h" /> @@ -132,6 +133,7 @@ <ClCompile Include="projectstore.cpp" /> <ClCompile Include="cache\cacheagent.cpp" /> <ClCompile Include="sos\sos.cpp" /> + <ClCompile Include="testing\httptest.cpp" /> <ClCompile Include="upstream\jupiter.cpp" /> <ClCompile Include="testing\launch.cpp" /> <ClCompile Include="cache\cachestore.cpp" /> @@ -146,6 +148,9 @@ <ProjectReference Include="..\zencore\zencore.vcxproj"> <Project>{d75bf9ab-c61e-4fff-ad59-1563430f05e2}</Project> </ProjectReference> + <ProjectReference Include="..\zenhttp\zenhttp.vcxproj"> + <Project>{8eeb3be5-7001-46bf-aafd-edb7558ac012}</Project> + </ProjectReference> <ProjectReference Include="..\zenstore\zenstore.vcxproj"> <Project>{26cbbaeb-14c1-4efc-877d-80f48215651c}</Project> </ProjectReference> @@ -153,6 +158,9 @@ <Project>{77f8315d-b21d-4db0-9a6f-2d3359f88a70}</Project> </ProjectReference> </ItemGroup> + <ItemGroup> + <None Include="xmake.lua" /> + </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters index ca16caf77..a86a6d96d 100644 --- a/zenserver/zenserver.vcxproj.filters +++ b/zenserver/zenserver.vcxproj.filters @@ -38,6 +38,7 @@ <ClInclude Include="upstream\upstreamcache.h"> <Filter>upstream</Filter> </ClInclude> + <ClInclude Include="testing\httptest.h" /> </ItemGroup> <ItemGroup> <ClCompile Include="zenserver.cpp" /> @@ -71,6 +72,7 @@ <ClCompile Include="upstream\upstreamcache.cpp"> <Filter>upstream</Filter> </ClCompile> + <ClCompile Include="testing\httptest.cpp" /> </ItemGroup> <ItemGroup> <Filter Include="cache"> @@ -89,4 +91,7 @@ <UniqueIdentifier>{303c28c2-3607-4ef4-89bd-e3618fe37e74}</UniqueIdentifier> </Filter> </ItemGroup> + <ItemGroup> + <None Include="xmake.lua" /> + </ItemGroup> </Project>
\ No newline at end of file |