// Copyright Epic Games, Inc. All Rights Reserved. #include "cache_cmd.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END namespace zen { namespace { IoBuffer CreateRandomBlob(uint64_t Size) { static uint64_t Seed{0x7CEBF54E45B9F5D1}; auto Next = [](uint64_t& seed) { uint64_t z = (seed += UINT64_C(0x9E3779B97F4A7C15)); z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9); z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB); return z ^ (z >> 31); }; IoBuffer Data(Size); uint64_t* DataPtr = reinterpret_cast(Data.MutableData()); while (Size > sizeof(uint64_t)) { *DataPtr++ = Next(Seed); Size -= sizeof(uint64_t); } uint64_t ByteNext = Next(Seed); uint8_t* ByteDataPtr = reinterpret_cast(DataPtr); while (Size > 0) { *ByteDataPtr++ = static_cast(ByteNext & 0xff); ByteNext >>= 8; Size--; } return Data; }; CompressedBuffer CompressBlob(IoBuffer&& Buffer) { return CompressedBuffer::Compress(SharedBuffer(Buffer), OodleCompressor::Mermaid, OodleCompressionLevel::SuperFast); } } // namespace DropCommand::DropCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), ""); m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), ""); m_Options.parse_positional({"namespace", "bucket"}); } DropCommand::~DropCommand() = default; int DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) { return 0; } m_HostName = ResolveTargetHostSpec(m_HostName); if (m_HostName.empty()) { throw zen::OptionParseException("unable to resolve server specification"); } if (m_NamespaceName.empty()) { throw zen::OptionParseException("Drop command requires a namespace"); } cpr::Session Session; if (m_BucketName.empty()) { ZEN_CONSOLE("Dropping cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); Session.SetUrl({fmt::format("{}/z$/{}", m_HostName, m_NamespaceName)}); } else { ZEN_CONSOLE("Dropping cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); Session.SetUrl({fmt::format("{}/z$/{}/{}", m_HostName, m_NamespaceName, m_BucketName)}); } cpr::Response Result = Session.Delete(); if (zen::IsHttpSuccessCode(Result.status_code)) { ZEN_CONSOLE("OK: drop succeeded"); return 0; } if (Result.status_code) { ZEN_ERROR("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { ZEN_ERROR("Drop failed: {}", Result.error.message); } return 1; } CacheInfoCommand::CacheInfoCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), ""); m_Options.add_option("", "", "bucketsizes", "Comma delimited list of bucket names to get size info from, * to get info on all buckets", cxxopts::value(m_SizeInfoBucketNames), ""); m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), ""); m_Options.add_option("", "", "bucketsize", "Show detailed bucket size info", cxxopts::value(m_BucketSizeInfo), ""); m_Options.parse_positional({"namespace", "bucket"}); } CacheInfoCommand::~CacheInfoCommand() = default; int CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) { return 0; } m_HostName = ResolveTargetHostSpec(m_HostName); if (m_HostName.empty()) { throw zen::OptionParseException("unable to resolve server specification"); } cpr::Session Session; Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); if (m_HostName.empty()) { if (!m_SizeInfoBucketNames.empty()) { throw zen::OptionParseException("--bucketsizes option needs a --namespace"); } if (m_BucketSizeInfo) { throw zen::OptionParseException("--bucketsizes option needs a --namespace and a --bucket"); } ZEN_CONSOLE("Info on cache from '{}'", m_HostName); Session.SetUrl({fmt::format("{}/z$", m_HostName)}); } else if (m_BucketName.empty()) { if (m_BucketSizeInfo) { throw zen::OptionParseException("--bucketsizes option needs a --bucket"); } ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); Session.SetUrl({fmt::format("{}/z$/{}", m_HostName, m_NamespaceName)}); } else { if (!m_SizeInfoBucketNames.empty()) { throw zen::OptionParseException("--bucketsizes option can not be used together with --bucket option"); } ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); Session.SetUrl({fmt::format("{}/z$/{}/{}", m_HostName, m_NamespaceName, m_BucketName)}); } cpr::Parameters Parameters; if (!m_SizeInfoBucketNames.empty()) { Parameters.Add({"bucketsizes", m_SizeInfoBucketNames}); } if (m_BucketSizeInfo) { Parameters.Add({"bucketsize", "true"}); } Session.SetParameters(Parameters); cpr::Response Result = Session.Get(); if (zen::IsHttpSuccessCode(Result.status_code)) { ZEN_CONSOLE("{}", Result.text); return 0; } if (Result.status_code) { ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { ZEN_ERROR("Info failed: {}", Result.error.message); } return 1; } CacheStatsCommand::CacheStatsCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); } CacheStatsCommand::~CacheStatsCommand() = default; int CacheStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) { return 0; } m_HostName = ResolveTargetHostSpec(m_HostName); if (m_HostName.empty()) { throw zen::OptionParseException("unable to resolve server specification"); } cpr::Session Session; Session.SetUrl({fmt::format("{}/stats/z$", m_HostName)}); Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); cpr::Response Result = Session.Get(); if (zen::IsHttpSuccessCode(Result.status_code)) { ZEN_CONSOLE("{}", Result.text); return 0; } if (Result.status_code) { ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { ZEN_ERROR("Info failed: {}", Result.error.message); } return 1; } CacheDetailsCommand::CacheDetailsCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); m_Options.add_option("", "c", "csv", "Info on csv format", cxxopts::value(m_CSV), ""); m_Options.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "
"); m_Options.add_option("", "a", "attachmentdetails", "Get detailed information about attachments", cxxopts::value(m_AttachmentDetails), ""); m_Options.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), ""); m_Options.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), ""); m_Options.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), ""); } CacheDetailsCommand::~CacheDetailsCommand() = default; int CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) { return 0; } m_HostName = ResolveTargetHostSpec(m_HostName); if (m_HostName.empty()) { throw zen::OptionParseException("unable to resolve server specification"); } cpr::Session Session; cpr::Parameters Parameters; if (m_Details) { Parameters.Add({"details", "true"}); } if (m_AttachmentDetails) { Parameters.Add({"attachmentdetails", "true"}); } if (m_CSV) { Parameters.Add({"csv", "true"}); } else { Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); } if (!m_ValueKey.empty()) { if (m_Namespace.empty() || m_Bucket.empty()) { ZEN_ERROR("Provide namespace and bucket name"); ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); return 1; } Session.SetUrl({fmt::format("{}/z$/details$/{}/{}/{}", m_HostName, m_Namespace, m_Bucket, m_ValueKey)}); } else if (!m_Bucket.empty()) { if (m_Namespace.empty()) { ZEN_ERROR("Provide namespace name"); ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); return 1; } Session.SetUrl({fmt::format("{}/z$/details$/{}/{}", m_HostName, m_Namespace, m_Bucket)}); } else if (!m_Namespace.empty()) { Session.SetUrl({fmt::format("{}/z$/details$/{}", m_HostName, m_Namespace)}); } else { Session.SetUrl({fmt::format("{}/z$/details$", m_HostName)}); } Session.SetParameters(Parameters); cpr::Response Result = Session.Get(); if (zen::IsHttpSuccessCode(Result.status_code)) { ZEN_CONSOLE("{}", Result.text); return 0; } if (Result.status_code) { ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { ZEN_ERROR("Info failed: {}", Result.error.message); } return 1; } CacheGenerateCommand::CacheGenerateCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); m_Options .add_option("", "n", "namespace", "Namespace to generate cache values/records for", cxxopts::value(m_Namespace), ""); m_Options.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), ""); m_Options.add_option("", "", "count", "Number of cache values/records to generate", cxxopts::value(m_Count), ""); m_Options.add_option("", "", "min-size", "Minimum size of cache value/attachments", cxxopts::value(m_MinSize), ""); m_Options.add_option("", "", "max-size", "Maximum size of cache value/attachments", cxxopts::value(m_MaxSize), ""); m_Options.add_option("", "", "min-attachments", "Minimum number of attachments when creating record based values", cxxopts::value(m_MinAttachmentCount), ""); m_Options.add_option("", "", "max-attachments", "Minimum number of attachments when creating record based values, 0 to only create cache values", cxxopts::value(m_MaxAttachmentCount), ""); m_Options.parse_positional({"namespace", "bucket", "count"}); m_Options.positional_help("namespace bucket count"); } CacheGenerateCommand::~CacheGenerateCommand() = default; int CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) { return 0; } m_HostName = ResolveTargetHostSpec(m_HostName); if (m_HostName.empty()) { throw zen::OptionParseException("unable to resolve server specification"); } if (m_MaxSize == 0 && m_MinSize == 0) { m_MinSize = 17; if (m_MaxAttachmentCount == 0) { // For cache values this max size will result in about 0.5% of values being saved as loose file in a cache bucket m_MaxSize = 65u * 1024u; } else { // For cache records this max size will result in about 0.5% of attachments begin saved as loose files in cas m_MaxSize = 768u * 1024u; } } eastl::vector> Variations; eastl::vector SizeRanges; SizeRanges.push_back(m_MinSize); Variations.push_back(std::uniform_int_distribution(0, m_MinSize - 1)); while (SizeRanges.back() < m_MaxSize) { SizeRanges.push_back(SizeRanges.back() * 2); Variations.push_back(std::uniform_int_distribution(0, SizeRanges.back() - 1)); } if (SizeRanges.back() > m_MaxSize) { SizeRanges.back() = m_MaxSize; Variations.push_back(std::uniform_int_distribution(0, m_MaxSize - 1)); } std::random_device RandomDevice; std::mt19937 Generator(RandomDevice()); std::uniform_int_distribution SizeRangeDistribution(0, SizeRanges.size() - 1); eastl::vector Sizes; Sizes.reserve(m_Count); for (uint64_t n = 0; n != m_Count; ++n) { uint64_t Range = SizeRangeDistribution(Generator); uint64_t Size = SizeRanges[Range]; uint64_t Variation = Variations[Range](Generator); Sizes.push_back(Size + Variation); } std::uniform_int_distribution KeyDistribution; HttpClient Http(m_HostName); auto GeneratePutCacheValueRequest([this, &KeyDistribution, &Generator](eastl::span BatchSizes, uint64_t RequestIndex) { cacherequests::PutCacheValuesRequest Request({.AcceptMagic = kCbPkgMagic, .Namespace = m_Namespace}); for (std::uint64_t ValueSize : BatchSizes) { uint64_t KeyBase = KeyDistribution(Generator); std::string KeyString = fmt::format("{}-{}-{}", RequestIndex, KeyBase, ValueSize); IoHash ValueKey = IoHash::HashBuffer(KeyString.c_str(), KeyString.length()); Request.Requests.emplace_back(cacherequests::PutCacheValueRequest{.Key = {.Bucket = m_Bucket, .Hash = ValueKey}, .Body = CompressBlob(CreateRandomBlob(ValueSize))}); } return Request; }); auto GeneratePutCacheRecordRequest([this, &KeyDistribution, &Generator](eastl::span BatchSizes, uint64_t RequestIndex) { cacherequests::PutCacheRecordsRequest Request({.AcceptMagic = kCbPkgMagic, .Namespace = m_Namespace}); uint64_t KeyBase = KeyDistribution(Generator); std::string RecordKeyString = fmt::format("{}-{}-{}", RequestIndex, KeyBase, BatchSizes.size()); IoHash RecordKey = IoHash::HashBuffer(RecordKeyString.c_str(), RecordKeyString.length()); Request.Requests.emplace_back(cacherequests::PutCacheRecordRequest{.Key = {.Bucket = m_Bucket, .Hash = RecordKey}}); for (std::uint64_t ValueSize : BatchSizes) { Request.Requests.back().Values.push_back({.Id = Oid::NewOid(), .Body = CompressBlob(CreateRandomBlob(ValueSize))}); } return Request; }); WorkerThreadPool WorkerPool(gsl::narrow(Max((std::thread::hardware_concurrency() / 2u), 2u))); Latch WorkLatch(1); std::uniform_int_distribution SizeCountDistribution(m_MaxAttachmentCount > 0 ? 0 : 1, m_MaxAttachmentCount > 0 ? m_MaxAttachmentCount : 8); std::size_t Offset = 0; uint64_t RequestIndex = 0; while (Offset < Sizes.size()) { size_t SizeCount = SizeCountDistribution(Generator); eastl::span BatchSizes = eastl::span(Sizes).subspan(Offset, Min(Max(SizeCount, 1u), Sizes.size() - Offset)); WorkLatch.AddCount(1); WorkerPool.ScheduleWork([&, BatchSizes, RequestIndex]() { auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); CbPackage Package; if (m_MaxAttachmentCount > 0 && SizeCount > 0) { auto Request = GeneratePutCacheRecordRequest(BatchSizes, RequestIndex); ZEN_ASSERT(Request.Format(Package)); } else { auto Request = GeneratePutCacheValueRequest(BatchSizes, RequestIndex); ZEN_ASSERT(Request.Format(Package)); } if (HttpClient::Response Response = Http.Post("/z$/$rpc", Package, HttpClient::Accept(ZenContentType::kCbPackage)); !Response) { ZEN_CONSOLE("{}", Response.ErrorMessage(fmt::format("{}: ", RequestIndex))); } }); Offset += BatchSizes.size(); RequestIndex++; } WorkLatch.CountDown(); while (!WorkLatch.Wait(1000)) { ZEN_INFO("Creating data, {} requests remaining", WorkLatch.Remaining()); } return 0; } CacheGetCommand::CacheGetCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); m_Options .add_option("", "n", "namespace", "Namespace to generate cache values/records for", cxxopts::value(m_Namespace), ""); m_Options.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), ""); m_Options.add_option("", "v", "valuekey", "Cache entry iohash id", cxxopts::value(m_ValueKey), ""); m_Options.add_option("", "a", "attachmenthash", "For a cache entry record, get a particular attachment based on the 'RawHash'", cxxopts::value(m_AttachmentHash), ""); m_Options.add_option("", "o", "output-path", "File path for output data", cxxopts::value(m_OutputPath), ""); m_Options.add_option("", "t", "text", "Ouput content of cache entry record as text", cxxopts::value(m_AsText), ""); m_Options .add_option("", "d", "decompress", "Decompress data when applicable. Default = true", cxxopts::value(m_Decompress), ""); m_Options.parse_positional({"namespace", "bucket", "valuekey", "attachmenthash"}); m_Options.positional_help("namespace bucket valuekey attachmenthash"); } CacheGetCommand::~CacheGetCommand() = default; int CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions); using namespace std::literals; if (!ParseOptions(argc, argv)) { return 0; } m_HostName = ResolveTargetHostSpec(m_HostName); if (m_HostName.empty()) { throw zen::OptionParseException("unable to resolve server specification"); } if (m_Namespace.empty()) { throw zen::OptionParseException("cache-get command requires a namespace"); } if (m_Bucket.empty()) { throw zen::OptionParseException("cache-get command requires a bucket"); } if (m_ValueKey.empty()) { throw zen::OptionParseException("cache-get command requires a value key"); } IoHash ValueId; if (!IoHash::TryParse(m_ValueKey, ValueId)) { throw zen::OptionParseException("cache-get --valuekey option requires a valid IoHash string"); } IoHash AttachmentHash; if (!m_AttachmentHash.empty()) { if (!IoHash::TryParse(m_AttachmentHash, AttachmentHash)) { throw zen::OptionParseException("cache-get --attachmenthash option requires a valid IoHash string"); } } HttpClient Http(m_HostName); std::filesystem::path TargetPath; if (!m_OutputPath.empty()) { TargetPath = std::filesystem::path(m_OutputPath); if (std::filesystem::is_directory(TargetPath)) { TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } else { CreateDirectories(TargetPath.parent_path()); } } if (TargetPath.empty()) { TargetPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey); if (AttachmentHash != IoHash::Zero) { Url = fmt::format("{}/{}", Url, AttachmentHash); } if (HttpClient::Response Result = Http.Download(Url, std::filesystem::temp_directory_path()); Result) { auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer { IoHash RawHash; uint64_t RawSize; if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize)) { return Compressed.Decompress().AsIoBuffer(); }; return std::move(Buffer); }; IoBuffer ChunkData = m_Decompress ? TryDecompress(Result.ResponsePayload) : Result.ResponsePayload; if (m_AsText) { std::string StringData = Result.ToText(); if (m_OutputPath.empty()) { ZEN_CONSOLE("{}", StringData); } else { WriteFile(TargetPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length())); ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), TargetPath, ToString(ChunkData.GetContentType())); } } else { if (!MoveToFile(TargetPath, ChunkData)) { WriteFile(TargetPath, ChunkData); } ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), TargetPath, ToString(ChunkData.GetContentType())); } } else { Result.ThrowError("Failed to fetch data"sv); } return 0; } } // namespace zen