diff options
| author | Martin Ridgers <[email protected]> | 2022-03-16 14:58:34 +0100 |
|---|---|---|
| committer | Martin Ridgers <[email protected]> | 2022-03-16 14:58:34 +0100 |
| commit | ef2f48fbfa3dc5d7a61baf6ccfd6e41b6c3eb895 (patch) | |
| tree | 2afa4ea4993869a10e37baa6062f8f4f3eca91f0 /zenserver | |
| parent | Corrected linux install (diff) | |
| parent | Fixed typo (diff) | |
| download | zen-ef2f48fbfa3dc5d7a61baf6ccfd6e41b6c3eb895.tar.xz zen-ef2f48fbfa3dc5d7a61baf6ccfd6e41b6c3eb895.zip | |
Merge branch 'dashboard-zipfs'
Diffstat (limited to 'zenserver')
| -rw-r--r-- | zenserver/experimental/frontend.cpp | 119 | ||||
| -rw-r--r-- | zenserver/experimental/frontend.h | 24 | ||||
| -rw-r--r-- | zenserver/frontend/frontend.cpp | 250 | ||||
| -rw-r--r-- | zenserver/frontend/frontend.h | 25 | ||||
| -rw-r--r-- | zenserver/frontend/html/index.html | 59 | ||||
| -rw-r--r-- | zenserver/frontend/zipfs.cpp | 170 | ||||
| -rw-r--r-- | zenserver/frontend/zipfs.h | 24 | ||||
| -rw-r--r-- | zenserver/zenserver.cpp | 2 |
8 files changed, 529 insertions, 144 deletions
diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp deleted file mode 100644 index 4bd3ec90a..000000000 --- a/zenserver/experimental/frontend.cpp +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "frontend.h" - -#include <zencore/filesystem.h> -#include <zencore/string.h> - -namespace zen { - -namespace html { - - constexpr std::string_view Index = R"( -<!DOCTYPE html> -<html> -<head> -<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous"> -<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.min.js" integrity="sha384-skAcpIdS7UcVUC05LJ9Dxay8AXcDYfBJqt1CJ85S/CFujBsIzCIv+l9liuYLaMQ/" crossorigin="anonymous"></script> -<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css"> -<style type="text/css"> -body { - background-color: #fafafa; -} -</style> -<script type="text/javascript"> - const getCacheStats = () => { - const opts = { headers: { "Accept": "application/json" } }; - fetch("/stats/z$", opts) - .then(response => { - if (!response.ok) { - throw Error(response.statusText); - } - return response.json(); - }) - .then(json => { - document.getElementById("status").innerHTML = "connected" - document.getElementById("stats").innerHTML = JSON.stringify(json, null, 4); - }) - .catch(error => { - document.getElementById("status").innerHTML = "disconnected" - document.getElementById("stats").innerHTML = "" - console.log(error); - }) - .finally(() => { - window.setTimeout(getCacheStats, 1000); - }); - }; - getCacheStats(); -</script> -</head> -<body> - <div class="container"> - <div class="row"> - <div class="text-center mt-5"> - <pre> -__________ _________ __ -\____ / ____ ____ / _____/_/ |_ ____ _______ ____ - / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ - / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ -/_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > - \/ \/ \/ \/ \/ - </pre> - <pre id="status"/> - </div> - </div> - <div class="row"> - <pre class="mb-0">Z$:</pre> - <pre id="stats"></pre> - <div> - </div> -</body> -</html> -)"; - -} // namespace html - -HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) -{ -} - -HttpFrontendService::~HttpFrontendService() -{ -} - -const char* -HttpFrontendService::BaseUri() const -{ - return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp -} - -void -HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) -{ - using namespace std::literals; - - if (m_Directory.empty()) - { - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, html::Index); - } - else - { - std::string_view Uri = Request.RelativeUri(); - std::filesystem::path RelPath{Uri.empty() ? "index.html" : Uri}; - std::filesystem::path AbsPath = m_Directory / RelPath; - - FileContents File = ReadFile(AbsPath); - - if (!File.ErrorCode) - { - // TODO: Map file extension to MIME type - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, File.Data[0]); - } - else - { - return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Ooops!"sv); - } - } -} - -} // namespace zen diff --git a/zenserver/experimental/frontend.h b/zenserver/experimental/frontend.h deleted file mode 100644 index 2ae20e940..000000000 --- a/zenserver/experimental/frontend.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zenhttp/httpserver.h> - -#include <filesystem> - -namespace zen { - -class HttpFrontendService final : public zen::HttpService -{ -public: - HttpFrontendService(std::filesystem::path Directory); - virtual ~HttpFrontendService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; - -private: - std::filesystem::path m_Directory; -}; - -} // namespace zen diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp new file mode 100644 index 000000000..b87d7e313 --- /dev/null +++ b/zenserver/frontend/frontend.cpp @@ -0,0 +1,250 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "frontend.h" + +#include <zencore/endian.h> +#include <zencore/filesystem.h> +#include <zencore/string.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#if ZEN_PLATFORM_WINDOWS +# include <Windows.h> +#endif +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +static IoBuffer FindZipFsInBinary(const IoBuffer& BinBuffer) +{ + if (BinBuffer.GetSize() < 4) + { + return {}; + } + + uintptr_t Cursor = uintptr_t(BinBuffer.GetData()); + size_t BinSize = 0; + + uint32_t Magic = *(uint32_t*)(BinBuffer.GetData()); +#if ZEN_PLATFORM_LINUX + if (Magic == 0x464c457f) + { + struct Elf64Header + { + char Ident[16]; + uint16_t Type; + uint16_t Machine; + uint32_t Version; + uint64_t Entry; + uint64_t ProgHeaderOffset; + uint64_t SectionHeaderOffset; + uint32_t Flags; + uint16_t EhSize; + uint16_t ProgHeaderEntrySize; + uint16_t ProgHeaderCount; + uint16_t SectionHeaderEntrySize; + uint16_t SectionHeaderCount; + uint16_t SectionStringIndex; + }; + + struct SectionHeader + { + uint32_t NameIndex; + uint32_t Type; + uint64_t Flags; + uint64_t Address; + uint64_t Offset; + uint64_t Size; + uint64_t _Other[3]; + }; + + const auto* Elf = (Elf64Header*)Cursor; + if (Elf->Ident[4] != 0x02) // Elf64 + { + return {}; + } + + const auto* Section = (SectionHeader*)(Cursor + Elf->SectionHeaderOffset); + + /* + size_t BinSize = 0; + for (int i = 0, n = Elf->SectionHeaderCount; i < n; ++i, ++Section) + { + uint32_t SectionEnd = Section->Offset + Section->Size; + BinSize = (SectionEnd > BinSize) ? SectionEnd : BinSize; + } + */ + + // What if the section headers aren't the last thing in the fiile though? + BinSize = Elf->SectionHeaderEntrySize; + BinSize *= Elf->SectionHeaderCount; + BinSize += Elf->SectionHeaderOffset; + } +#elif ZEN_PLATFORM_WINDOWS + if ((Magic & 0xffff) == 0x5a4d) + { + const auto* Dos = (IMAGE_DOS_HEADER*)Cursor; + const auto* Nt = (IMAGE_NT_HEADERS64*)(Cursor + Dos->e_lfanew); + const auto* Section = (IMAGE_SECTION_HEADER*)(uintptr_t(&Nt->OptionalHeader) + Nt->FileHeader.SizeOfOptionalHeader); + + for (int i = 0, n = Nt->FileHeader.NumberOfSections; i < n; ++i, ++Section) + { + uint32_t SectionEnd = Section->PointerToRawData + Section->SizeOfRawData; + BinSize = (SectionEnd > BinSize) ? SectionEnd : BinSize; + } + } +#elif ZEN_PLATFORM_MAC + if (Magic == 0xbebafeca) + { + struct MachInt32 + { + operator uint32_t () const { return ByteSwap(Value); } + uint32_t Value; + }; + + struct MachFatArch + { + MachInt32 CpuType; + MachInt32 SubType; + MachInt32 Offset; + MachInt32 Size; + MachInt32 Alignment; + }; + + struct MachFatHeader + { + uint32_t Magic; + MachInt32 NumArchs; + MachFatArch Archs[]; + }; + + const auto* Header = (MachFatHeader*)Cursor; + for (int i = 0, n = Header->NumArchs; i < n; ++i) + { + const MachFatArch* Arch = Header->Archs + i; + uint32_t ArchEnd = Arch->Offset + Arch->Size; + BinSize = (ArchEnd > BinSize) ? ArchEnd : BinSize; + } + } +#endif // win/linux/mac + + if (!BinSize || BinSize > BinBuffer.GetSize()) + { + return {}; + } + + return IoBuffer(BinBuffer, BinSize); +} + +//////////////////////////////////////////////////////////////////////////////// +HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) +: m_Directory(Directory) +{ + std::filesystem::path SelfPath = GetRunningExecutablePath(); + + // Locate a .zip file appended onto the end of this binary + IoBuffer SelfBuffer = IoBufferBuilder::MakeFromFile(SelfPath); + IoBuffer SelfTailBuffer = FindZipFsInBinary(SelfBuffer); + if (SelfTailBuffer) + { + m_ZipFs = ZipFs(std::move(SelfTailBuffer)); + } + +#if ZEN_BUILD_DEBUG + if (!Directory.empty()) + { + return; + } + + std::error_code ErrorCode; + for (auto Path = SelfPath.parent_path(); !Path.empty(); Path = Path.parent_path()) + { + if (!std::filesystem::is_regular_file(Path / "xmake.lua", ErrorCode)) + { + continue; + } + + auto HtmlDir = (Path / __FILE__).parent_path() / "html"; + if (std::filesystem::is_directory(HtmlDir, ErrorCode)) + { + m_Directory = HtmlDir; + } + break; + } +#endif +} + +//////////////////////////////////////////////////////////////////////////////// +HttpFrontendService::~HttpFrontendService() +{ +} + +//////////////////////////////////////////////////////////////////////////////// +const char* +HttpFrontendService::BaseUri() const +{ + return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp +} + +//////////////////////////////////////////////////////////////////////////////// +void +HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) +{ + using namespace std::literals; + + std::string_view Uri = Request.RelativeUri(); + for (; Uri[0] == '/'; Uri = Uri.substr(1)); + if (Uri.empty()) + { + Uri = "index.html"sv; + } + + // Dismiss if the URI contains .. anywhere to prevent arbitrary file reads + if (Uri.find("..") != Uri.npos) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + return; + } + + // Map the file extension to a MIME type. To keep things constrained, only a + // small subset of file extensions is allowed. + HttpContentType ContentType = HttpContentType::kCOUNT; + size_t DotIndex = Uri.rfind("."); + if (DotIndex != Uri.npos) + { + const std::string_view DotExt = Uri.substr(DotIndex); + if (DotExt == ".html") ContentType = HttpContentType::kHTML; + else if (DotExt == ".js") ContentType = HttpContentType::kJSON; + else if (DotExt == ".css") ContentType = HttpContentType::kCSS; + else if (DotExt == ".png") ContentType = HttpContentType::kPNG; + else if (DotExt == ".ico") ContentType = HttpContentType::kIcon; + } + + if (ContentType == HttpContentType::kCOUNT) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + return; + } + + // The given content directory overrides any zip-fs discovered in the binary + if (!m_Directory.empty()) + { + FileContents File = ReadFile(m_Directory / Uri); + if (!File.ErrorCode) + { + Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]); + return; + } + } + + IoBuffer FileBuffer = m_ZipFs.GetFile(Uri); + if (FileBuffer) + { + Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + return; + } + + Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); +} + +} // namespace zen diff --git a/zenserver/frontend/frontend.h b/zenserver/frontend/frontend.h new file mode 100644 index 000000000..bf5298169 --- /dev/null +++ b/zenserver/frontend/frontend.h @@ -0,0 +1,25 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenhttp/httpserver.h> +#include "zipfs.h" + +#include <filesystem> + +namespace zen { + +class HttpFrontendService final : public zen::HttpService +{ +public: + HttpFrontendService(std::filesystem::path Directory); + virtual ~HttpFrontendService(); + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + ZipFs m_ZipFs; + std::filesystem::path m_Directory; +}; + +} // namespace zen diff --git a/zenserver/frontend/html/index.html b/zenserver/frontend/html/index.html new file mode 100644 index 000000000..252ee621e --- /dev/null +++ b/zenserver/frontend/html/index.html @@ -0,0 +1,59 @@ +<!DOCTYPE html> +<html> +<head> + <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous"> + <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.min.js" integrity="sha384-skAcpIdS7UcVUC05LJ9Dxay8AXcDYfBJqt1CJ85S/CFujBsIzCIv+l9liuYLaMQ/" crossorigin="anonymous"></script> + <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css"> + <style type="text/css"> + body { + background-color: #fafafa; + } + </style> + <script type="text/javascript"> + const getCacheStats = () => { + const opts = { headers: { "Accept": "application/json" } }; + fetch("/stats/z$", opts) + .then(response => { + if (!response.ok) { + throw Error(response.statusText); + } + return response.json(); + }) + .then(json => { + document.getElementById("status").innerHTML = "connected" + document.getElementById("stats").innerHTML = JSON.stringify(json, null, 4); + }) + .catch(error => { + document.getElementById("status").innerHTML = "disconnected" + document.getElementById("stats").innerHTML = "" + console.log(error); + }) + .finally(() => { + window.setTimeout(getCacheStats, 1000); + }); + }; + getCacheStats(); + </script> +</head> +<body> + <div class="container"> + <div class="row"> + <div class="text-center mt-5"> + <pre> +__________ _________ __ +\____ / ____ ____ / _____/_/ |_ ____ _______ ____ + / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ + / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ +/_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > + \/ \/ \/ \/ \/ + </pre> + <pre id="status"/> + </div> + </div> + <div class="row"> + <pre class="mb-0">Z$:</pre> + <pre id="stats"></pre> + <div> + </div> +</body> +</html> diff --git a/zenserver/frontend/zipfs.cpp b/zenserver/frontend/zipfs.cpp new file mode 100644 index 000000000..5fb9d0177 --- /dev/null +++ b/zenserver/frontend/zipfs.cpp @@ -0,0 +1,170 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zipfs.h" + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +namespace { + +#if ZEN_COMPILER_MSC +# pragma warning(push) +# pragma warning(disable : 4200) +#endif + +using ZipInt16 = uint16_t; + +struct ZipInt32 +{ + operator uint32_t () const { return *(uint32_t*)Parts; } + uint16_t Parts[2]; +}; + +struct EocdRecord +{ + enum : uint32_t { + Magic = 0x0605'4b50, + }; + ZipInt32 Signature; + ZipInt16 ThisDiskIndex; + ZipInt16 CdStartDiskIndex; + ZipInt16 CdRecordThisDiskCount; + ZipInt16 CdRecordCount; + ZipInt32 CdSize; + ZipInt32 CdOffset; + ZipInt16 CommentSize; + char Comment[]; +}; + +struct CentralDirectoryRecord +{ + enum : uint32_t { + Magic = 0x0201'4b50, + }; + + ZipInt32 Signature; + ZipInt16 VersionMadeBy; + ZipInt16 VersionRequired; + ZipInt16 Flags; + ZipInt16 CompressionMethod; + ZipInt16 LastModTime; + ZipInt16 LastModDate; + ZipInt32 Crc32; + ZipInt32 CompressedSize; + ZipInt32 OriginalSize; + ZipInt16 FileNameLength; + ZipInt16 ExtraFieldLength; + ZipInt16 CommentLength; + ZipInt16 DiskIndex; + ZipInt16 InternalFileAttr; + ZipInt32 ExternalFileAttr; + ZipInt32 Offset; + char FileName[]; +}; + +struct LocalFileHeader +{ + enum : uint32_t { + Magic = 0x0304'4b50, + }; + + ZipInt32 Signature; + ZipInt16 VersionRequired; + ZipInt16 Flags; + ZipInt16 CompressionMethod; + ZipInt16 LastModTime; + ZipInt16 LastModDate; + ZipInt32 Crc32; + ZipInt32 CompressedSize; + ZipInt32 OriginalSize; + ZipInt16 FileNameLength; + ZipInt16 ExtraFieldLength; + char FileName[]; +}; + +#if ZEN_COMPILER_MSC +# pragma warning(pop) +#endif + +} // namespace + + + +////////////////////////////////////////////////////////////////////////// +ZipFs::ZipFs(IoBuffer&& Buffer) +{ + MemoryView View = Buffer.GetView(); + + uint8_t* Cursor = (uint8_t*)(View.GetData()) + View.GetSize(); + if (View.GetSize() < sizeof(EocdRecord)) + { + return; + } + + const auto* EocdCursor = (EocdRecord*)(Cursor - sizeof(EocdRecord)); + + // It is more correct to search backwards for EocdRecord::Magic as the + // comment can be of a variable length. But here we're not going to support + // zip files with comments. + if (EocdCursor->Signature != EocdRecord::Magic) + { + return; + } + + // Zip64 isn't supported either + if (EocdCursor->ThisDiskIndex == 0xffff) + { + return; + } + + Cursor -= View.GetSize(); + + const auto* CdCursor = (CentralDirectoryRecord*)(Cursor + EocdCursor->CdOffset); + for (int i = 0, n = EocdCursor->CdRecordCount; i < n; ++i) + { + const CentralDirectoryRecord& Cd = *CdCursor; + + bool Acceptable = true; + Acceptable &= (Cd.OriginalSize > 0); // has some content + Acceptable &= (Cd.CompressionMethod == 0); // is stored uncomrpessed + if (Acceptable) + { + const uint8_t* Lfh = Cursor + Cd.Offset; + if (uintptr_t(Lfh - Cursor) < View.GetSize()) + { + std::string_view FileName(Cd.FileName, Cd.FileNameLength); + m_Files.insert(std::make_pair(FileName, FileItem{Lfh, size_t(0)})); + } + } + + uint32_t ExtraBytes = Cd.FileNameLength + Cd.ExtraFieldLength + Cd.CommentLength; + CdCursor = (CentralDirectoryRecord*)(Cd.FileName + ExtraBytes); + } + + m_Buffer = std::move(Buffer); +} + +////////////////////////////////////////////////////////////////////////// +IoBuffer ZipFs::GetFile(const std::string_view& FileName) const +{ + FileMap::iterator Iter = m_Files.find(FileName); + if (Iter == m_Files.end()) + { + return{}; + } + + FileItem& Item = Iter->second; + if (Item.GetSize() > 0) + { + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); + } + + const auto* Lfh = (LocalFileHeader*)(Item.GetData()); + Item = MemoryView( + Lfh->FileName + Lfh->FileNameLength + Lfh->ExtraFieldLength, + Lfh->OriginalSize + ); + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); +} + +} // namespace zen diff --git a/zenserver/frontend/zipfs.h b/zenserver/frontend/zipfs.h new file mode 100644 index 000000000..a304e9ff5 --- /dev/null +++ b/zenserver/frontend/zipfs.h @@ -0,0 +1,24 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/iobuffer.h> + +#include <unordered_map> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +class ZipFs +{ +public: + ZipFs() = default; + ZipFs(IoBuffer&& Buffer); + IoBuffer GetFile(const std::string_view& FileName) const; + +private: + using FileItem = MemoryView; + using FileMap = std::unordered_map<std::string_view, FileItem>; + FileMap mutable m_Files; + IoBuffer m_Buffer; +}; + +} // namespace zen diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 576c88cb8..a684272c4 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -106,7 +106,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "cache/structuredcachestore.h" #include "compute/apply.h" #include "diag/diagsvcs.h" -#include "experimental/frontend.h" +#include "frontend/frontend.h" #include "experimental/usnjournal.h" #include "monitoring/httpstats.h" #include "monitoring/httpstatus.h" |