From 645ce60a493e7c1af24f245c56b6c3fca3e3f2b2 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Sun, 3 Oct 2021 09:38:09 +0200 Subject: Fixed missing content type. --- zenhttp/httpserver.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp index cfd1463ba..3a746fd51 100644 --- a/zenhttp/httpserver.cpp +++ b/zenhttp/httpserver.cpp @@ -74,6 +74,7 @@ static constinit uint32_t HashCompactBinary = HashStringDjb2("application/x-u static constinit uint32_t HashCompactBinaryPackage = HashStringDjb2("application/x-ue-cbpkg"sv); static constinit uint32_t HashCompactBinaryPackageOffer = HashStringDjb2("application/x-ue-offer"sv); static constinit uint32_t HashCompressedBinary = HashStringDjb2("application/x-ue-comp"sv); +static constinit uint32_t HashHtml = HashStringDjb2("text/html"sv); std::once_flag InitContentTypeLookup; @@ -88,7 +89,8 @@ struct HashedTypeEntry {HashJson, HttpContentType::kJSON}, {HashYaml, HttpContentType::kYAML}, {HashText, HttpContentType::kText}, - {HashCompressedBinary, HttpContentType::kCompressedBinary}}; + {HashCompressedBinary, HttpContentType::kCompressedBinary}, + {HashHtml, HttpContentType::kHTML}}; HttpContentType ParseContentTypeImpl(const std::string_view& ContentTypeString) -- cgit v1.2.3 From c058fb0a11f9843d30bfdc3cd5d1d3a6bd26c3d5 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Sun, 3 Oct 2021 16:25:43 +0200 Subject: Added support for SkipData cache policy. --- zenserver-test/zenserver-test.cpp | 106 ++++++++++++++++++++++++++++++++---- zenserver/cache/structuredcache.cpp | 27 ++++++++- 2 files changed, 119 insertions(+), 14 deletions(-) diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 0e5e73ffc..fe21aa834 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1448,16 +1448,21 @@ TEST_CASE("zcache.policy") return Buf; }; - auto GeneratePackage = [](zen::IoHash& OutAttachmentKey) -> zen::CbPackage { + auto GeneratePackage = [](zen::IoHash& OutRecordKey, zen::IoHash& OutAttachmentKey) -> zen::CbPackage { auto Data = zen::SharedBuffer::Clone(zen::MakeMemoryView({1, 2, 3, 4, 5, 6, 7, 8, 9})); auto CompressedData = zen::CompressedBuffer::Compress(Data); OutAttachmentKey = zen::IoHash::FromBLAKE3(CompressedData.GetRawHash()); - zen::CbWriter Obj; - Obj.BeginObject("obj"sv); - Obj.AddBinaryAttachment("data", OutAttachmentKey); - Obj.EndObject(); + + zen::CbWriter Writer; + Writer.BeginObject("obj"sv); + Writer.AddBinaryAttachment("data", OutAttachmentKey); + Writer.EndObject(); + CbObject CacheRecord = Writer.Save().AsObject(); + + OutRecordKey = IoHash::HashBuffer(CacheRecord.GetBuffer().GetView()); + zen::CbPackage Package; - Package.SetObject(Obj.Save().AsObject()); + Package.SetObject(CacheRecord); Package.AddAttachment(zen::CbAttachment(CompressedData)); return Package; @@ -1587,7 +1592,8 @@ TEST_CASE("zcache.policy") LocalCfg.Spawn(LocalInst); zen::IoHash Key; - zen::CbPackage Package = GeneratePackage(Key); + zen::IoHash PayloadId; + zen::CbPackage Package = GeneratePackage(Key, PayloadId); auto Buf = ToBuffer(Package); // Store package upstream @@ -1623,7 +1629,8 @@ TEST_CASE("zcache.policy") LocalCfg.Spawn(LocalInst); zen::IoHash Key; - zen::CbPackage Package = GeneratePackage(Key); + zen::IoHash PayloadId; + zen::CbPackage Package = GeneratePackage(Key, PayloadId); auto Buf = ToBuffer(Package); // Store packge locally @@ -1659,7 +1666,8 @@ TEST_CASE("zcache.policy") LocalCfg.Spawn(LocalInst); zen::IoHash Key; - zen::CbPackage Package = GeneratePackage(Key); + zen::IoHash PayloadId; + zen::CbPackage Package = GeneratePackage(Key, PayloadId); auto Buf = ToBuffer(Package); // Store package locally and upstream @@ -1692,7 +1700,8 @@ TEST_CASE("zcache.policy") LocalCfg.Spawn(LocalInst); zen::IoHash Key; - zen::CbPackage Package = GeneratePackage(Key); + zen::IoHash PayloadId; + zen::CbPackage Package = GeneratePackage(Key, PayloadId); auto Buf = ToBuffer(Package); // Store package locally @@ -1748,7 +1757,8 @@ TEST_CASE("zcache.policy") LocalCfg.Spawn(LocalInst); zen::IoHash Key; - zen::CbPackage Package = GeneratePackage(Key); + zen::IoHash PayloadId; + zen::CbPackage Package = GeneratePackage(Key, PayloadId); auto Buf = ToBuffer(Package); // Store package upstream @@ -1791,6 +1801,80 @@ TEST_CASE("zcache.policy") CHECK(Package.GetAttachments().size() != 0); } } + + SUBCASE("skip - 'data' returns empty cache record/payload") + { + ZenConfig Cfg = ZenConfig::New(); + ZenServerInstance Instance(TestEnv); + const auto Bucket = "test"sv; + + Cfg.Spawn(Instance); + + zen::IoHash Key; + zen::IoHash PayloadId; + zen::CbPackage Package = GeneratePackage(Key, PayloadId); + auto Buf = ToBuffer(Package); + + // Store package + { + cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(Cfg.BaseUri, Bucket, Key)}, + cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()}, + cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}}); + CHECK(Result.status_code == 201); + } + + // Get package + { + cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)}, + cpr::Header{{"Accept", "application/x-ue-cbpkg"}}); + CHECK(Result.status_code == 200); + CHECK(Result.text.size() == 0); + } + + // Get record + { + cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)}, + cpr::Header{{"Accept", "application/x-ue-cbobject"}}); + CHECK(Result.status_code == 200); + CHECK(Result.text.size() == 0); + } + + // Get payload + { + cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key, PayloadId)}, + cpr::Header{{"Accept", "application/x-ue-comp"}}); + CHECK(Result.status_code == 200); + CHECK(Result.text.size() == 0); + } + } + + SUBCASE("skip - 'data' returns empty binary value") + { + ZenConfig Cfg = ZenConfig::New(); + ZenServerInstance Instance(TestEnv); + const auto Bucket = "test"sv; + + Cfg.Spawn(Instance); + + zen::IoHash Key; + auto BinaryValue = GenerateData(1024, Key); + + // Store binary cache value + { + cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(Cfg.BaseUri, Bucket, Key)}, + cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()}, + cpr::Header{{"Content-Type", "application/octet-stream"}}); + CHECK(Result.status_code == 201); + } + + // Get package + { + cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)}, + cpr::Header{{"Accept", "application/octet-stream"}}); + CHECK(Result.status_code == 200); + CHECK(Result.text.size() == 0); + } + } } struct RemoteExecutionRequest diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 8ab0276c5..fe9fdd415 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -418,10 +418,17 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (ValidationResult != CbValidateError::None) { ZEN_WARN("GET - '{}/{}' '{}' FAILED, invalid compact binary object", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); + m_CacheStats.MissCount++; return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv); } - const bool SkipAttachments = zen::CachePolicy::SkipAttachments == (Policy & zen::CachePolicy::SkipAttachments); + if ((Policy & CachePolicy::SkipData) == CachePolicy::SkipData) + { + m_CacheStats.HitCount++; + return Request.WriteResponse(HttpResponseCode::OK); + } + + const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; uint32_t AttachmentCount = 0; uint32_t ValidCount = 0; uint64_t AttachmentBytes = 0ull; @@ -487,7 +494,14 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request m_CacheStats.UpstreamHitCount++; } - Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); + if ((Policy & CachePolicy::SkipData) == CachePolicy::SkipData) + { + Request.WriteResponse(HttpResponseCode::OK); + } + else + { + Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); + } } } @@ -740,7 +754,14 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques m_CacheStats.UpstreamHitCount++; } - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload); + if ((Policy & CachePolicy::SkipData) == CachePolicy::SkipData) + { + Request.WriteResponse(HttpResponseCode::OK); + } + else + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload); + } } void -- cgit v1.2.3