embed html frontend content as binary compiled data (#559)

- Feature: Added xmake task `updatefrontend` which updates the zip file containing the frontend html (`/src/zenserver/frontend/html.zip`)
- Improvement: The frontend html content is no longer appended at the end of the executable which prevented signing, instead it is compiled in from the `/src/zenserver/frontend/html.zip` archive
- Improvement: MacOS now does ad-hoc code signing by default when issuing `xmake bundle`, signing with proper cert is done on CI builds

9 files changed, 168 insertions, 31 deletions

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 658b6b648..e6616aa25 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -152,7 +152,7 @@ jobs:
 
     - name: Bundle
       run: |
-        xmake bundle -v -y
+        xmake bundle -v -y --codesignidentity="Developer ID Application"
       env:
         VCPKG_ROOT: ${{ github.workspace }}/.vcpkg
 
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index 0960882d6..bdaf672f2 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -227,7 +227,7 @@ jobs:
     - name: Bundle
       if: ${{ matrix.config == 'release' }}
       run: |
-        xmake bundle -v -y
+        xmake bundle -v -y --codesignidentity="Developer ID Application"
       env:
         VCPKG_ROOT: ${{ github.workspace }}/.vcpkg
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba8d18127..0af3ad5e5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,7 @@
 ##
+- Feature: Added xmake task `updatefrontend` which updates the zip file containing the frontend html (`/src/zenserver/frontend/html.zip`)
+- Improvement: The frontend html content is no longer appended at the end of the executable which prevented signing, instead it is compiled in from the `/src/zenserver/frontend/html.zip` archive
+- Improvement: MacOS now does ad-hoc code signing by default when issuing `xmake bundle`, signing with proper cert is done on CI builds
 
 ## 0.2.35
 - Bugfix: Fix timeout calculation for semtimedop call
diff --git a/scripts/bundle.lua b/scripts/bundle.lua
index a8a6b973a..207122345 100644
--- a/scripts/bundle.lua
+++ b/scripts/bundle.lua
@@ -156,21 +156,6 @@ local function _find_vcpkg_binary(triple, port, binary)
 end
 
 --------------------------------------------------------------------------------
-local function _append_content_zip(bin_path)
-    local zip_path = "build/frontend.zip"
-    local content_dir = "src/zenserver/frontend/html/"
-    _zip(true, zip_path, content_dir)
-
-    zip_file = io.open(zip_path, "rb")
-    local zip_data = zip_file:read("*all")
-    zip_file:close()
-
-    bin_file = io.open(bin_path, "ab")
-    bin_file:write(zip_data)
-    bin_file:close()
-end
-
---------------------------------------------------------------------------------
 local function main_windows()
     import("core.base.option")
 
@@ -183,8 +168,6 @@ local function main_windows()
 
     _build("x64", false, config_args)
 
-    _append_content_zip("build/windows/x64/release/zenserver.exe")
-
     local crashpad_handler_path = _find_vcpkg_binary(
         "x64-windows-static",
         "sentry-native",
@@ -200,7 +183,7 @@ local function main_windows()
 end
 
 --------------------------------------------------------------------------------
-local function main_mac()
+local function main_mac(signidentity)
     -- Build and universalify
     _build("x86_64", false, "--target_minver=10.15")
     _build("arm64", false, "--target_minver=10.15")
@@ -211,25 +194,43 @@ local function main_mac()
         "-create",
         "-output", "build/macosx/universal/release/zenserver",
         "build/macosx/x86_64/release/zenserver",
-        "build/macosx/arm64/release/zenserver"
-    )
+        "build/macosx/arm64/release/zenserver")
     if ret > 0 then
         raise("Failed creating universal zenserver binary")
     end
 
-    _append_content_zip("build/macosx/universal/release/zenserver")
-
     ret = _exec(
         "lipo",
         "-create",
         "-output", "build/macosx/universal/release/zen",
         "build/macosx/x86_64/release/zen",
-        "build/macosx/arm64/release/zen"
-    )
+        "build/macosx/arm64/release/zen")
     if ret > 0 then
         raise("Failed creating universal zen binary")
     end
 
+    ret = _exec("codesign",
+                "-s",
+                signidentity,
+                "-f",
+                "-v",
+                "--deep",
+                "build/macosx/universal/release/zenserver")
+    if ret > 0 then
+        raise("Failed signing universal zenserver binary")
+    end
+
+    ret = _exec("codesign",
+                "-s",
+                signidentity,
+                "-f",
+                "-v",
+                "--deep",
+                "build/macosx/universal/release/zen")
+    if ret > 0 then
+        raise("Failed signing universal zen binary")
+    end
+
     -- At the time of writing vcpkg does not support sentry-native on arm64. Once
     -- it does we can create a univeral binary for this. For now just bundle x64
     local crashpad_handler_path = _find_vcpkg_binary(
@@ -258,8 +259,6 @@ local function main_linux()
         "crashpad_handler")
     --]]
 
-    _append_content_zip("build/linux/x86_64/release/zenserver")
-
     _zip(false,
         "build/zenserver-linux.zip",
         "build/linux/x86_64/release/zenserver",
@@ -268,13 +267,21 @@ local function main_linux()
 end
 
 --------------------------------------------------------------------------------
+
+import("core.base.option")
+
 function main()
     if is_host("windows") then
         return main_windows()
     end
 
     if is_host("mac") then
-        return main_mac()
+        signidentity = option.get("codesignidentity")
+        if (signidentity == nil or signidentity == '') then
+            signidentity = '-'
+        end
+
+        return main_mac(signidentity)
     end
 
     if is_host("linux") then
diff --git a/scripts/updatefrontend.lua b/scripts/updatefrontend.lua
new file mode 100644
index 000000000..ab37819d7
--- /dev/null
+++ b/scripts/updatefrontend.lua
@@ -0,0 +1,111 @@
+-- Copyright Epic Games, Inc. All Rights Reserved.
+
+--------------------------------------------------------------------------------
+local function _exec(cmd, ...)
+    local args = {}
+    for _, arg in pairs({...}) do
+        if arg then
+            table.insert(args, arg)
+        end
+    end
+
+    print("--", cmd, table.unpack(args))
+    local ret = os.execv(cmd, args)
+    print()
+    return ret
+end
+
+--------------------------------------------------------------------------------
+local function _zip(store_only, zip_path, ...)
+    -- Here's the rules; if len(...) is 1 and it is a dir then create a zip with
+    -- archive paths like this;
+    --
+    --   glob(foo/bar/**) -> foo/bar/abc, foo/bar/dir/123 -> zip(abc, dir/123)
+    --
+    -- Otherwise assume ... is file paths and add without leading directories;
+    --
+    --   foo/abc, bar/123 -> zip(abc, 123)
+
+    zip_path = path.absolute(zip_path)
+    os.tryrm(zip_path)
+
+    local inputs = {...}
+
+    local source_dir = nil
+    if #inputs == 1 and os.isdir(inputs[1]) then
+        source_dir = inputs[1]
+    end
+
+    import("detect.tools.find_7z")
+    local cmd_7z = find_7z()
+    if cmd_7z then
+        input_paths = {}
+        if source_dir then
+            -- Suffixing a directory path with a "/." will have 7z set the path
+            -- for archived files relative to that directory.
+            input_paths = { path.join(source_dir, ".") }
+        else
+            for _, input_path in pairs(inputs) do
+                -- If there is a "/./" anywhere in file paths then 7z drops all
+                -- directory information and just archives the file by name
+                input_path = path.relative(input_path, ".")
+                if input_path:sub(2,2) ~= ":" then
+                    input_path = "./"..input_path
+                end
+                table.insert(input_paths, input_path)
+            end
+        end
+
+        compression_level = "-mx1"
+        if store_only then
+            compression_level = "-mx0"
+        end
+
+        local ret = _exec(cmd_7z, "a", compression_level, zip_path, table.unpack(input_paths))
+        if ret > 0 then
+            raise("Received error from 7z")
+        end
+        return
+    end
+
+    print("7z not found, falling back to zip")
+
+    import("detect.tools.find_zip")
+    zip_cmd = find_zip()
+    if zip_cmd then
+        local input_paths = inputs
+        local cwd = os.curdir()
+        if source_dir then
+            os.cd(source_dir)
+            input_paths = { "." }
+        end
+
+        compression_level = "-1"
+        if store_only then
+            compression_level = "-0"
+        end
+
+        local strip_leading_path = nil
+        if not source_dir then
+            strip_leading_path = "--junk-paths"
+        end
+
+        local ret = _exec(zip_cmd, "-r", compression_level, strip_leading_path, zip_path, table.unpack(input_paths))
+        if ret > 0 then
+            raise("Received error from zip")
+        end
+
+        os.cd(cwd)
+        return
+    end
+    print("zip not found")
+
+    raise("Unable to find a suitable zip tool")
+end
+
+--------------------------------------------------------------------------------
+function main()
+    local zip_path = "src/zenserver/frontend/html.zip"
+    local content_dir = "src/zenserver/frontend/html/"
+    _zip(true, zip_path, content_dir)
+end
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 8c8e5cb9c..9bc408711 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -14,6 +14,9 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #endif
 ZEN_THIRD_PARTY_INCLUDES_END
 
+static unsigned char gHtmlZipData[] = {
+#include <html.zip.h>
+};
 namespace zen {
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -22,8 +25,8 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
 	std::filesystem::path SelfPath = GetRunningExecutablePath();
 
 	// Locate a .zip file appended onto the end of this binary
-	IoBuffer SelfBuffer = IoBufferBuilder::MakeFromFile(SelfPath);
-	m_ZipFs				= ZipFs(std::move(SelfBuffer));
+	IoBuffer HtmlZipDataBuffer(IoBuffer::Wrap, gHtmlZipData, sizeof(gHtmlZipData) - 1);
+	m_ZipFs = ZipFs(std::move(HtmlZipDataBuffer));
 
 	if (m_Directory.empty() && !m_ZipFs)
 	{
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
new file mode 100644
index 000000000..fa2f2febf
--- /dev/null
+++ b/src/zenserver/frontend/html.zip
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 127213ebd..c42f305ee 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -9,7 +9,9 @@ target("zenserver")
              "zenutil",
              "zenvfs")
     add_headerfiles("**.h")
+    add_rules("utils.bin2c", {extensions = {".zip"}})
     add_files("**.cpp")
+    add_files("frontend/*.zip")
     add_files("zenserver.cpp", {unity_ignored  = true })
     add_includedirs(".")
     set_symbols("debug")
diff --git a/xmake.lua b/xmake.lua
index 3cf4815f9..57e2f3390 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -166,6 +166,7 @@ task("bundle")
         description = "Create Zip bundle from binaries",
         options = {
             {nil, "withtrace", "k", nil, "Compiles with trace support"},
+            {nil, "codesignidentity", "v", nil, "Code signing identity"},
         }
     }
     on_run(function ()
@@ -173,6 +174,16 @@ task("bundle")
         bundle()
     end)
 
+task("updatefrontend")
+    set_menu {
+        usage = "xmake updatefrontend",
+        description = "Create Zip of the frontend/html folder for bundling with zenserver executable",
+    }
+    on_run(function()
+        import("scripts.updatefrontend")
+        updatefrontend()
+    end)
+
 task("precommit")
     set_menu {
         usage = "xmake precommit",