diff options
| author | Stefan Boberg <[email protected]> | 2021-05-11 13:05:39 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2021-05-11 13:05:39 +0200 |
| commit | f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa (patch) | |
| tree | 1daf7621e110d48acd5e12e3073ce48ef0dd11b2 /zenserver | |
| download | zen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.tar.xz zen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.zip | |
Adding zenservice code
Diffstat (limited to 'zenserver')
36 files changed, 7567 insertions, 0 deletions
diff --git a/zenserver/admin/admin.h b/zenserver/admin/admin.h new file mode 100644 index 000000000..3bb8a9158 --- /dev/null +++ b/zenserver/admin/admin.h @@ -0,0 +1,18 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/httpserver.h> + +class HttpAdminService : public zen::HttpService +{ +public: + HttpAdminService() = default; + ~HttpAdminService() = default; + + virtual const char* BaseUri() const override { return "/admin/"; } + + virtual void HandleRequest(zen::HttpServerRequest& Request) override { ZEN_UNUSED(Request); } + +private: +}; diff --git a/zenserver/cache/cacheagent.cpp b/zenserver/cache/cacheagent.cpp new file mode 100644 index 000000000..f4d1cabe6 --- /dev/null +++ b/zenserver/cache/cacheagent.cpp @@ -0,0 +1,5 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "cacheagent.h" + +#include <gsl/gsl-lite.hpp> diff --git a/zenserver/cache/cacheagent.h b/zenserver/cache/cacheagent.h new file mode 100644 index 000000000..145d0f79f --- /dev/null +++ b/zenserver/cache/cacheagent.h @@ -0,0 +1,9 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +class CacheAgent +{ +public: +private: +}; diff --git a/zenserver/cache/cachestore.cpp b/zenserver/cache/cachestore.cpp new file mode 100644 index 000000000..fc218de6b --- /dev/null +++ b/zenserver/cache/cachestore.cpp @@ -0,0 +1,1235 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "cachestore.h" + +#include <zencore/windows.h> + +#include <fmt/core.h> +#include <spdlog/spdlog.h> +#include <zencore/filesystem.h> +#include <zencore/iobuffer.h> +#include <zencore/string.h> +#include <zencore/thread.h> +#include <zenstore/cas.h> +#include <filesystem> +#include <gsl/gsl-lite.hpp> +#include <unordered_map> + +#include <atlfile.h> + +using namespace zen; + +namespace UE { + +static const uint32_t CRCTable[256] = { + 0x00000000, 0x04C11DB7, 0x09823B6E, 0x0D4326D9, 0x130476DC, 0x17C56B6B, 0x1A864DB2, 0x1E475005, 0x2608EDB8, 0x22C9F00F, 0x2F8AD6D6, + 0x2B4BCB61, 0x350C9B64, 0x31CD86D3, 0x3C8EA00A, 0x384FBDBD, 0x4C11DB70, 0x48D0C6C7, 0x4593E01E, 0x4152FDA9, 0x5F15ADAC, 0x5BD4B01B, + 0x569796C2, 0x52568B75, 0x6A1936C8, 0x6ED82B7F, 0x639B0DA6, 0x675A1011, 0x791D4014, 0x7DDC5DA3, 0x709F7B7A, 0x745E66CD, 0x9823B6E0, + 0x9CE2AB57, 0x91A18D8E, 0x95609039, 0x8B27C03C, 0x8FE6DD8B, 0x82A5FB52, 0x8664E6E5, 0xBE2B5B58, 0xBAEA46EF, 0xB7A96036, 0xB3687D81, + 0xAD2F2D84, 0xA9EE3033, 0xA4AD16EA, 0xA06C0B5D, 0xD4326D90, 0xD0F37027, 0xDDB056FE, 0xD9714B49, 0xC7361B4C, 0xC3F706FB, 0xCEB42022, + 0xCA753D95, 0xF23A8028, 0xF6FB9D9F, 0xFBB8BB46, 0xFF79A6F1, 0xE13EF6F4, 0xE5FFEB43, 0xE8BCCD9A, 0xEC7DD02D, 0x34867077, 0x30476DC0, + 0x3D044B19, 0x39C556AE, 0x278206AB, 0x23431B1C, 0x2E003DC5, 0x2AC12072, 0x128E9DCF, 0x164F8078, 0x1B0CA6A1, 0x1FCDBB16, 0x018AEB13, + 0x054BF6A4, 0x0808D07D, 0x0CC9CDCA, 0x7897AB07, 0x7C56B6B0, 0x71159069, 0x75D48DDE, 0x6B93DDDB, 0x6F52C06C, 0x6211E6B5, 0x66D0FB02, + 0x5E9F46BF, 0x5A5E5B08, 0x571D7DD1, 0x53DC6066, 0x4D9B3063, 0x495A2DD4, 0x44190B0D, 0x40D816BA, 0xACA5C697, 0xA864DB20, 0xA527FDF9, + 0xA1E6E04E, 0xBFA1B04B, 0xBB60ADFC, 0xB6238B25, 0xB2E29692, 0x8AAD2B2F, 0x8E6C3698, 0x832F1041, 0x87EE0DF6, 0x99A95DF3, 0x9D684044, + 0x902B669D, 0x94EA7B2A, 0xE0B41DE7, 0xE4750050, 0xE9362689, 0xEDF73B3E, 0xF3B06B3B, 0xF771768C, 0xFA325055, 0xFEF34DE2, 0xC6BCF05F, + 0xC27DEDE8, 0xCF3ECB31, 0xCBFFD686, 0xD5B88683, 0xD1799B34, 0xDC3ABDED, 0xD8FBA05A, 0x690CE0EE, 0x6DCDFD59, 0x608EDB80, 0x644FC637, + 0x7A089632, 0x7EC98B85, 0x738AAD5C, 0x774BB0EB, 0x4F040D56, 0x4BC510E1, 0x46863638, 0x42472B8F, 0x5C007B8A, 0x58C1663D, 0x558240E4, + 0x51435D53, 0x251D3B9E, 0x21DC2629, 0x2C9F00F0, 0x285E1D47, 0x36194D42, 0x32D850F5, 0x3F9B762C, 0x3B5A6B9B, 0x0315D626, 0x07D4CB91, + 0x0A97ED48, 0x0E56F0FF, 0x1011A0FA, 0x14D0BD4D, 0x19939B94, 0x1D528623, 0xF12F560E, 0xF5EE4BB9, 0xF8AD6D60, 0xFC6C70D7, 0xE22B20D2, + 0xE6EA3D65, 0xEBA91BBC, 0xEF68060B, 0xD727BBB6, 0xD3E6A601, 0xDEA580D8, 0xDA649D6F, 0xC423CD6A, 0xC0E2D0DD, 0xCDA1F604, 0xC960EBB3, + 0xBD3E8D7E, 0xB9FF90C9, 0xB4BCB610, 0xB07DABA7, 0xAE3AFBA2, 0xAAFBE615, 0xA7B8C0CC, 0xA379DD7B, 0x9B3660C6, 0x9FF77D71, 0x92B45BA8, + 0x9675461F, 0x8832161A, 0x8CF30BAD, 0x81B02D74, 0x857130C3, 0x5D8A9099, 0x594B8D2E, 0x5408ABF7, 0x50C9B640, 0x4E8EE645, 0x4A4FFBF2, + 0x470CDD2B, 0x43CDC09C, 0x7B827D21, 0x7F436096, 0x7200464F, 0x76C15BF8, 0x68860BFD, 0x6C47164A, 0x61043093, 0x65C52D24, 0x119B4BE9, + 0x155A565E, 0x18197087, 0x1CD86D30, 0x029F3D35, 0x065E2082, 0x0B1D065B, 0x0FDC1BEC, 0x3793A651, 0x3352BBE6, 0x3E119D3F, 0x3AD08088, + 0x2497D08D, 0x2056CD3A, 0x2D15EBE3, 0x29D4F654, 0xC5A92679, 0xC1683BCE, 0xCC2B1D17, 0xC8EA00A0, 0xD6AD50A5, 0xD26C4D12, 0xDF2F6BCB, + 0xDBEE767C, 0xE3A1CBC1, 0xE760D676, 0xEA23F0AF, 0xEEE2ED18, 0xF0A5BD1D, 0xF464A0AA, 0xF9278673, 0xFDE69BC4, 0x89B8FD09, 0x8D79E0BE, + 0x803AC667, 0x84FBDBD0, 0x9ABC8BD5, 0x9E7D9662, 0x933EB0BB, 0x97FFAD0C, 0xAFB010B1, 0xAB710D06, 0xA6322BDF, 0xA2F33668, 0xBCB4666D, + 0xB8757BDA, 0xB5365D03, 0xB1F740B4}; + +static const uint32_t CRCTablesSB8[8][256] = { + {0x00000000, 0xb71dc104, 0x6e3b8209, 0xd926430d, 0xdc760413, 0x6b6bc517, 0xb24d861a, 0x0550471e, 0xb8ed0826, 0x0ff0c922, 0xd6d68a2f, + 0x61cb4b2b, 0x649b0c35, 0xd386cd31, 0x0aa08e3c, 0xbdbd4f38, 0x70db114c, 0xc7c6d048, 0x1ee09345, 0xa9fd5241, 0xacad155f, 0x1bb0d45b, + 0xc2969756, 0x758b5652, 0xc836196a, 0x7f2bd86e, 0xa60d9b63, 0x11105a67, 0x14401d79, 0xa35ddc7d, 0x7a7b9f70, 0xcd665e74, 0xe0b62398, + 0x57abe29c, 0x8e8da191, 0x39906095, 0x3cc0278b, 0x8bdde68f, 0x52fba582, 0xe5e66486, 0x585b2bbe, 0xef46eaba, 0x3660a9b7, 0x817d68b3, + 0x842d2fad, 0x3330eea9, 0xea16ada4, 0x5d0b6ca0, 0x906d32d4, 0x2770f3d0, 0xfe56b0dd, 0x494b71d9, 0x4c1b36c7, 0xfb06f7c3, 0x2220b4ce, + 0x953d75ca, 0x28803af2, 0x9f9dfbf6, 0x46bbb8fb, 0xf1a679ff, 0xf4f63ee1, 0x43ebffe5, 0x9acdbce8, 0x2dd07dec, 0x77708634, 0xc06d4730, + 0x194b043d, 0xae56c539, 0xab068227, 0x1c1b4323, 0xc53d002e, 0x7220c12a, 0xcf9d8e12, 0x78804f16, 0xa1a60c1b, 0x16bbcd1f, 0x13eb8a01, + 0xa4f64b05, 0x7dd00808, 0xcacdc90c, 0x07ab9778, 0xb0b6567c, 0x69901571, 0xde8dd475, 0xdbdd936b, 0x6cc0526f, 0xb5e61162, 0x02fbd066, + 0xbf469f5e, 0x085b5e5a, 0xd17d1d57, 0x6660dc53, 0x63309b4d, 0xd42d5a49, 0x0d0b1944, 0xba16d840, 0x97c6a5ac, 0x20db64a8, 0xf9fd27a5, + 0x4ee0e6a1, 0x4bb0a1bf, 0xfcad60bb, 0x258b23b6, 0x9296e2b2, 0x2f2bad8a, 0x98366c8e, 0x41102f83, 0xf60dee87, 0xf35da999, 0x4440689d, + 0x9d662b90, 0x2a7bea94, 0xe71db4e0, 0x500075e4, 0x892636e9, 0x3e3bf7ed, 0x3b6bb0f3, 0x8c7671f7, 0x555032fa, 0xe24df3fe, 0x5ff0bcc6, + 0xe8ed7dc2, 0x31cb3ecf, 0x86d6ffcb, 0x8386b8d5, 0x349b79d1, 0xedbd3adc, 0x5aa0fbd8, 0xeee00c69, 0x59fdcd6d, 0x80db8e60, 0x37c64f64, + 0x3296087a, 0x858bc97e, 0x5cad8a73, 0xebb04b77, 0x560d044f, 0xe110c54b, 0x38368646, 0x8f2b4742, 0x8a7b005c, 0x3d66c158, 0xe4408255, + 0x535d4351, 0x9e3b1d25, 0x2926dc21, 0xf0009f2c, 0x471d5e28, 0x424d1936, 0xf550d832, 0x2c769b3f, 0x9b6b5a3b, 0x26d61503, 0x91cbd407, + 0x48ed970a, 0xfff0560e, 0xfaa01110, 0x4dbdd014, 0x949b9319, 0x2386521d, 0x0e562ff1, 0xb94beef5, 0x606dadf8, 0xd7706cfc, 0xd2202be2, + 0x653deae6, 0xbc1ba9eb, 0x0b0668ef, 0xb6bb27d7, 0x01a6e6d3, 0xd880a5de, 0x6f9d64da, 0x6acd23c4, 0xddd0e2c0, 0x04f6a1cd, 0xb3eb60c9, + 0x7e8d3ebd, 0xc990ffb9, 0x10b6bcb4, 0xa7ab7db0, 0xa2fb3aae, 0x15e6fbaa, 0xccc0b8a7, 0x7bdd79a3, 0xc660369b, 0x717df79f, 0xa85bb492, + 0x1f467596, 0x1a163288, 0xad0bf38c, 0x742db081, 0xc3307185, 0x99908a5d, 0x2e8d4b59, 0xf7ab0854, 0x40b6c950, 0x45e68e4e, 0xf2fb4f4a, + 0x2bdd0c47, 0x9cc0cd43, 0x217d827b, 0x9660437f, 0x4f460072, 0xf85bc176, 0xfd0b8668, 0x4a16476c, 0x93300461, 0x242dc565, 0xe94b9b11, + 0x5e565a15, 0x87701918, 0x306dd81c, 0x353d9f02, 0x82205e06, 0x5b061d0b, 0xec1bdc0f, 0x51a69337, 0xe6bb5233, 0x3f9d113e, 0x8880d03a, + 0x8dd09724, 0x3acd5620, 0xe3eb152d, 0x54f6d429, 0x7926a9c5, 0xce3b68c1, 0x171d2bcc, 0xa000eac8, 0xa550add6, 0x124d6cd2, 0xcb6b2fdf, + 0x7c76eedb, 0xc1cba1e3, 0x76d660e7, 0xaff023ea, 0x18ede2ee, 0x1dbda5f0, 0xaaa064f4, 0x738627f9, 0xc49be6fd, 0x09fdb889, 0xbee0798d, + 0x67c63a80, 0xd0dbfb84, 0xd58bbc9a, 0x62967d9e, 0xbbb03e93, 0x0cadff97, 0xb110b0af, 0x060d71ab, 0xdf2b32a6, 0x6836f3a2, 0x6d66b4bc, + 0xda7b75b8, 0x035d36b5, 0xb440f7b1}, + {0x00000000, 0xdcc119d2, 0x0f9ef2a0, 0xd35feb72, 0xa9212445, 0x75e03d97, 0xa6bfd6e5, 0x7a7ecf37, 0x5243488a, 0x8e825158, 0x5dddba2a, + 0x811ca3f8, 0xfb626ccf, 0x27a3751d, 0xf4fc9e6f, 0x283d87bd, 0x139b5110, 0xcf5a48c2, 0x1c05a3b0, 0xc0c4ba62, 0xbaba7555, 0x667b6c87, + 0xb52487f5, 0x69e59e27, 0x41d8199a, 0x9d190048, 0x4e46eb3a, 0x9287f2e8, 0xe8f93ddf, 0x3438240d, 0xe767cf7f, 0x3ba6d6ad, 0x2636a320, + 0xfaf7baf2, 0x29a85180, 0xf5694852, 0x8f178765, 0x53d69eb7, 0x808975c5, 0x5c486c17, 0x7475ebaa, 0xa8b4f278, 0x7beb190a, 0xa72a00d8, + 0xdd54cfef, 0x0195d63d, 0xd2ca3d4f, 0x0e0b249d, 0x35adf230, 0xe96cebe2, 0x3a330090, 0xe6f21942, 0x9c8cd675, 0x404dcfa7, 0x931224d5, + 0x4fd33d07, 0x67eebaba, 0xbb2fa368, 0x6870481a, 0xb4b151c8, 0xcecf9eff, 0x120e872d, 0xc1516c5f, 0x1d90758d, 0x4c6c4641, 0x90ad5f93, + 0x43f2b4e1, 0x9f33ad33, 0xe54d6204, 0x398c7bd6, 0xead390a4, 0x36128976, 0x1e2f0ecb, 0xc2ee1719, 0x11b1fc6b, 0xcd70e5b9, 0xb70e2a8e, + 0x6bcf335c, 0xb890d82e, 0x6451c1fc, 0x5ff71751, 0x83360e83, 0x5069e5f1, 0x8ca8fc23, 0xf6d63314, 0x2a172ac6, 0xf948c1b4, 0x2589d866, + 0x0db45fdb, 0xd1754609, 0x022aad7b, 0xdeebb4a9, 0xa4957b9e, 0x7854624c, 0xab0b893e, 0x77ca90ec, 0x6a5ae561, 0xb69bfcb3, 0x65c417c1, + 0xb9050e13, 0xc37bc124, 0x1fbad8f6, 0xcce53384, 0x10242a56, 0x3819adeb, 0xe4d8b439, 0x37875f4b, 0xeb464699, 0x913889ae, 0x4df9907c, + 0x9ea67b0e, 0x426762dc, 0x79c1b471, 0xa500ada3, 0x765f46d1, 0xaa9e5f03, 0xd0e09034, 0x0c2189e6, 0xdf7e6294, 0x03bf7b46, 0x2b82fcfb, + 0xf743e529, 0x241c0e5b, 0xf8dd1789, 0x82a3d8be, 0x5e62c16c, 0x8d3d2a1e, 0x51fc33cc, 0x98d88c82, 0x44199550, 0x97467e22, 0x4b8767f0, + 0x31f9a8c7, 0xed38b115, 0x3e675a67, 0xe2a643b5, 0xca9bc408, 0x165addda, 0xc50536a8, 0x19c42f7a, 0x63bae04d, 0xbf7bf99f, 0x6c2412ed, + 0xb0e50b3f, 0x8b43dd92, 0x5782c440, 0x84dd2f32, 0x581c36e0, 0x2262f9d7, 0xfea3e005, 0x2dfc0b77, 0xf13d12a5, 0xd9009518, 0x05c18cca, + 0xd69e67b8, 0x0a5f7e6a, 0x7021b15d, 0xace0a88f, 0x7fbf43fd, 0xa37e5a2f, 0xbeee2fa2, 0x622f3670, 0xb170dd02, 0x6db1c4d0, 0x17cf0be7, + 0xcb0e1235, 0x1851f947, 0xc490e095, 0xecad6728, 0x306c7efa, 0xe3339588, 0x3ff28c5a, 0x458c436d, 0x994d5abf, 0x4a12b1cd, 0x96d3a81f, + 0xad757eb2, 0x71b46760, 0xa2eb8c12, 0x7e2a95c0, 0x04545af7, 0xd8954325, 0x0bcaa857, 0xd70bb185, 0xff363638, 0x23f72fea, 0xf0a8c498, + 0x2c69dd4a, 0x5617127d, 0x8ad60baf, 0x5989e0dd, 0x8548f90f, 0xd4b4cac3, 0x0875d311, 0xdb2a3863, 0x07eb21b1, 0x7d95ee86, 0xa154f754, + 0x720b1c26, 0xaeca05f4, 0x86f78249, 0x5a369b9b, 0x896970e9, 0x55a8693b, 0x2fd6a60c, 0xf317bfde, 0x204854ac, 0xfc894d7e, 0xc72f9bd3, + 0x1bee8201, 0xc8b16973, 0x147070a1, 0x6e0ebf96, 0xb2cfa644, 0x61904d36, 0xbd5154e4, 0x956cd359, 0x49adca8b, 0x9af221f9, 0x4633382b, + 0x3c4df71c, 0xe08ceece, 0x33d305bc, 0xef121c6e, 0xf28269e3, 0x2e437031, 0xfd1c9b43, 0x21dd8291, 0x5ba34da6, 0x87625474, 0x543dbf06, + 0x88fca6d4, 0xa0c12169, 0x7c0038bb, 0xaf5fd3c9, 0x739eca1b, 0x09e0052c, 0xd5211cfe, 0x067ef78c, 0xdabfee5e, 0xe11938f3, 0x3dd82121, + 0xee87ca53, 0x3246d381, 0x48381cb6, 0x94f90564, 0x47a6ee16, 0x9b67f7c4, 0xb35a7079, 0x6f9b69ab, 0xbcc482d9, 0x60059b0b, 0x1a7b543c, + 0xc6ba4dee, 0x15e5a69c, 0xc924bf4e}, + {0x00000000, 0x87acd801, 0x0e59b103, 0x89f56902, 0x1cb26207, 0x9b1eba06, 0x12ebd304, 0x95470b05, 0x3864c50e, 0xbfc81d0f, 0x363d740d, + 0xb191ac0c, 0x24d6a709, 0xa37a7f08, 0x2a8f160a, 0xad23ce0b, 0x70c88a1d, 0xf764521c, 0x7e913b1e, 0xf93de31f, 0x6c7ae81a, 0xebd6301b, + 0x62235919, 0xe58f8118, 0x48ac4f13, 0xcf009712, 0x46f5fe10, 0xc1592611, 0x541e2d14, 0xd3b2f515, 0x5a479c17, 0xddeb4416, 0xe090153b, + 0x673ccd3a, 0xeec9a438, 0x69657c39, 0xfc22773c, 0x7b8eaf3d, 0xf27bc63f, 0x75d71e3e, 0xd8f4d035, 0x5f580834, 0xd6ad6136, 0x5101b937, + 0xc446b232, 0x43ea6a33, 0xca1f0331, 0x4db3db30, 0x90589f26, 0x17f44727, 0x9e012e25, 0x19adf624, 0x8ceafd21, 0x0b462520, 0x82b34c22, + 0x051f9423, 0xa83c5a28, 0x2f908229, 0xa665eb2b, 0x21c9332a, 0xb48e382f, 0x3322e02e, 0xbad7892c, 0x3d7b512d, 0xc0212b76, 0x478df377, + 0xce789a75, 0x49d44274, 0xdc934971, 0x5b3f9170, 0xd2caf872, 0x55662073, 0xf845ee78, 0x7fe93679, 0xf61c5f7b, 0x71b0877a, 0xe4f78c7f, + 0x635b547e, 0xeaae3d7c, 0x6d02e57d, 0xb0e9a16b, 0x3745796a, 0xbeb01068, 0x391cc869, 0xac5bc36c, 0x2bf71b6d, 0xa202726f, 0x25aeaa6e, + 0x888d6465, 0x0f21bc64, 0x86d4d566, 0x01780d67, 0x943f0662, 0x1393de63, 0x9a66b761, 0x1dca6f60, 0x20b13e4d, 0xa71de64c, 0x2ee88f4e, + 0xa944574f, 0x3c035c4a, 0xbbaf844b, 0x325aed49, 0xb5f63548, 0x18d5fb43, 0x9f792342, 0x168c4a40, 0x91209241, 0x04679944, 0x83cb4145, + 0x0a3e2847, 0x8d92f046, 0x5079b450, 0xd7d56c51, 0x5e200553, 0xd98cdd52, 0x4ccbd657, 0xcb670e56, 0x42926754, 0xc53ebf55, 0x681d715e, + 0xefb1a95f, 0x6644c05d, 0xe1e8185c, 0x74af1359, 0xf303cb58, 0x7af6a25a, 0xfd5a7a5b, 0x804356ec, 0x07ef8eed, 0x8e1ae7ef, 0x09b63fee, + 0x9cf134eb, 0x1b5decea, 0x92a885e8, 0x15045de9, 0xb82793e2, 0x3f8b4be3, 0xb67e22e1, 0x31d2fae0, 0xa495f1e5, 0x233929e4, 0xaacc40e6, + 0x2d6098e7, 0xf08bdcf1, 0x772704f0, 0xfed26df2, 0x797eb5f3, 0xec39bef6, 0x6b9566f7, 0xe2600ff5, 0x65ccd7f4, 0xc8ef19ff, 0x4f43c1fe, + 0xc6b6a8fc, 0x411a70fd, 0xd45d7bf8, 0x53f1a3f9, 0xda04cafb, 0x5da812fa, 0x60d343d7, 0xe77f9bd6, 0x6e8af2d4, 0xe9262ad5, 0x7c6121d0, + 0xfbcdf9d1, 0x723890d3, 0xf59448d2, 0x58b786d9, 0xdf1b5ed8, 0x56ee37da, 0xd142efdb, 0x4405e4de, 0xc3a93cdf, 0x4a5c55dd, 0xcdf08ddc, + 0x101bc9ca, 0x97b711cb, 0x1e4278c9, 0x99eea0c8, 0x0ca9abcd, 0x8b0573cc, 0x02f01ace, 0x855cc2cf, 0x287f0cc4, 0xafd3d4c5, 0x2626bdc7, + 0xa18a65c6, 0x34cd6ec3, 0xb361b6c2, 0x3a94dfc0, 0xbd3807c1, 0x40627d9a, 0xc7cea59b, 0x4e3bcc99, 0xc9971498, 0x5cd01f9d, 0xdb7cc79c, + 0x5289ae9e, 0xd525769f, 0x7806b894, 0xffaa6095, 0x765f0997, 0xf1f3d196, 0x64b4da93, 0xe3180292, 0x6aed6b90, 0xed41b391, 0x30aaf787, + 0xb7062f86, 0x3ef34684, 0xb95f9e85, 0x2c189580, 0xabb44d81, 0x22412483, 0xa5edfc82, 0x08ce3289, 0x8f62ea88, 0x0697838a, 0x813b5b8b, + 0x147c508e, 0x93d0888f, 0x1a25e18d, 0x9d89398c, 0xa0f268a1, 0x275eb0a0, 0xaeabd9a2, 0x290701a3, 0xbc400aa6, 0x3becd2a7, 0xb219bba5, + 0x35b563a4, 0x9896adaf, 0x1f3a75ae, 0x96cf1cac, 0x1163c4ad, 0x8424cfa8, 0x038817a9, 0x8a7d7eab, 0x0dd1a6aa, 0xd03ae2bc, 0x57963abd, + 0xde6353bf, 0x59cf8bbe, 0xcc8880bb, 0x4b2458ba, 0xc2d131b8, 0x457de9b9, 0xe85e27b2, 0x6ff2ffb3, 0xe60796b1, 0x61ab4eb0, 0xf4ec45b5, + 0x73409db4, 0xfab5f4b6, 0x7d192cb7}, + {0x00000000, 0xb79a6ddc, 0xd9281abc, 0x6eb27760, 0x054cf57c, 0xb2d698a0, 0xdc64efc0, 0x6bfe821c, 0x0a98eaf9, 0xbd028725, 0xd3b0f045, + 0x642a9d99, 0x0fd41f85, 0xb84e7259, 0xd6fc0539, 0x616668e5, 0xa32d14f7, 0x14b7792b, 0x7a050e4b, 0xcd9f6397, 0xa661e18b, 0x11fb8c57, + 0x7f49fb37, 0xc8d396eb, 0xa9b5fe0e, 0x1e2f93d2, 0x709de4b2, 0xc707896e, 0xacf90b72, 0x1b6366ae, 0x75d111ce, 0xc24b7c12, 0xf146e9ea, + 0x46dc8436, 0x286ef356, 0x9ff49e8a, 0xf40a1c96, 0x4390714a, 0x2d22062a, 0x9ab86bf6, 0xfbde0313, 0x4c446ecf, 0x22f619af, 0x956c7473, + 0xfe92f66f, 0x49089bb3, 0x27baecd3, 0x9020810f, 0x526bfd1d, 0xe5f190c1, 0x8b43e7a1, 0x3cd98a7d, 0x57270861, 0xe0bd65bd, 0x8e0f12dd, + 0x39957f01, 0x58f317e4, 0xef697a38, 0x81db0d58, 0x36416084, 0x5dbfe298, 0xea258f44, 0x8497f824, 0x330d95f8, 0x559013d1, 0xe20a7e0d, + 0x8cb8096d, 0x3b2264b1, 0x50dce6ad, 0xe7468b71, 0x89f4fc11, 0x3e6e91cd, 0x5f08f928, 0xe89294f4, 0x8620e394, 0x31ba8e48, 0x5a440c54, + 0xedde6188, 0x836c16e8, 0x34f67b34, 0xf6bd0726, 0x41276afa, 0x2f951d9a, 0x980f7046, 0xf3f1f25a, 0x446b9f86, 0x2ad9e8e6, 0x9d43853a, + 0xfc25eddf, 0x4bbf8003, 0x250df763, 0x92979abf, 0xf96918a3, 0x4ef3757f, 0x2041021f, 0x97db6fc3, 0xa4d6fa3b, 0x134c97e7, 0x7dfee087, + 0xca648d5b, 0xa19a0f47, 0x1600629b, 0x78b215fb, 0xcf287827, 0xae4e10c2, 0x19d47d1e, 0x77660a7e, 0xc0fc67a2, 0xab02e5be, 0x1c988862, + 0x722aff02, 0xc5b092de, 0x07fbeecc, 0xb0618310, 0xded3f470, 0x694999ac, 0x02b71bb0, 0xb52d766c, 0xdb9f010c, 0x6c056cd0, 0x0d630435, + 0xbaf969e9, 0xd44b1e89, 0x63d17355, 0x082ff149, 0xbfb59c95, 0xd107ebf5, 0x669d8629, 0x1d3de6a6, 0xaaa78b7a, 0xc415fc1a, 0x738f91c6, + 0x187113da, 0xafeb7e06, 0xc1590966, 0x76c364ba, 0x17a50c5f, 0xa03f6183, 0xce8d16e3, 0x79177b3f, 0x12e9f923, 0xa57394ff, 0xcbc1e39f, + 0x7c5b8e43, 0xbe10f251, 0x098a9f8d, 0x6738e8ed, 0xd0a28531, 0xbb5c072d, 0x0cc66af1, 0x62741d91, 0xd5ee704d, 0xb48818a8, 0x03127574, + 0x6da00214, 0xda3a6fc8, 0xb1c4edd4, 0x065e8008, 0x68ecf768, 0xdf769ab4, 0xec7b0f4c, 0x5be16290, 0x355315f0, 0x82c9782c, 0xe937fa30, + 0x5ead97ec, 0x301fe08c, 0x87858d50, 0xe6e3e5b5, 0x51798869, 0x3fcbff09, 0x885192d5, 0xe3af10c9, 0x54357d15, 0x3a870a75, 0x8d1d67a9, + 0x4f561bbb, 0xf8cc7667, 0x967e0107, 0x21e46cdb, 0x4a1aeec7, 0xfd80831b, 0x9332f47b, 0x24a899a7, 0x45cef142, 0xf2549c9e, 0x9ce6ebfe, + 0x2b7c8622, 0x4082043e, 0xf71869e2, 0x99aa1e82, 0x2e30735e, 0x48adf577, 0xff3798ab, 0x9185efcb, 0x261f8217, 0x4de1000b, 0xfa7b6dd7, + 0x94c91ab7, 0x2353776b, 0x42351f8e, 0xf5af7252, 0x9b1d0532, 0x2c8768ee, 0x4779eaf2, 0xf0e3872e, 0x9e51f04e, 0x29cb9d92, 0xeb80e180, + 0x5c1a8c5c, 0x32a8fb3c, 0x853296e0, 0xeecc14fc, 0x59567920, 0x37e40e40, 0x807e639c, 0xe1180b79, 0x568266a5, 0x383011c5, 0x8faa7c19, + 0xe454fe05, 0x53ce93d9, 0x3d7ce4b9, 0x8ae68965, 0xb9eb1c9d, 0x0e717141, 0x60c30621, 0xd7596bfd, 0xbca7e9e1, 0x0b3d843d, 0x658ff35d, + 0xd2159e81, 0xb373f664, 0x04e99bb8, 0x6a5becd8, 0xddc18104, 0xb63f0318, 0x01a56ec4, 0x6f1719a4, 0xd88d7478, 0x1ac6086a, 0xad5c65b6, + 0xc3ee12d6, 0x74747f0a, 0x1f8afd16, 0xa81090ca, 0xc6a2e7aa, 0x71388a76, 0x105ee293, 0xa7c48f4f, 0xc976f82f, 0x7eec95f3, 0x151217ef, + 0xa2887a33, 0xcc3a0d53, 0x7ba0608f}, + {0x00000000, 0x8d670d49, 0x1acf1a92, 0x97a817db, 0x8383f420, 0x0ee4f969, 0x994ceeb2, 0x142be3fb, 0x0607e941, 0x8b60e408, 0x1cc8f3d3, + 0x91affe9a, 0x85841d61, 0x08e31028, 0x9f4b07f3, 0x122c0aba, 0x0c0ed283, 0x8169dfca, 0x16c1c811, 0x9ba6c558, 0x8f8d26a3, 0x02ea2bea, + 0x95423c31, 0x18253178, 0x0a093bc2, 0x876e368b, 0x10c62150, 0x9da12c19, 0x898acfe2, 0x04edc2ab, 0x9345d570, 0x1e22d839, 0xaf016503, + 0x2266684a, 0xb5ce7f91, 0x38a972d8, 0x2c829123, 0xa1e59c6a, 0x364d8bb1, 0xbb2a86f8, 0xa9068c42, 0x2461810b, 0xb3c996d0, 0x3eae9b99, + 0x2a857862, 0xa7e2752b, 0x304a62f0, 0xbd2d6fb9, 0xa30fb780, 0x2e68bac9, 0xb9c0ad12, 0x34a7a05b, 0x208c43a0, 0xadeb4ee9, 0x3a435932, + 0xb724547b, 0xa5085ec1, 0x286f5388, 0xbfc74453, 0x32a0491a, 0x268baae1, 0xabeca7a8, 0x3c44b073, 0xb123bd3a, 0x5e03ca06, 0xd364c74f, + 0x44ccd094, 0xc9abdddd, 0xdd803e26, 0x50e7336f, 0xc74f24b4, 0x4a2829fd, 0x58042347, 0xd5632e0e, 0x42cb39d5, 0xcfac349c, 0xdb87d767, + 0x56e0da2e, 0xc148cdf5, 0x4c2fc0bc, 0x520d1885, 0xdf6a15cc, 0x48c20217, 0xc5a50f5e, 0xd18eeca5, 0x5ce9e1ec, 0xcb41f637, 0x4626fb7e, + 0x540af1c4, 0xd96dfc8d, 0x4ec5eb56, 0xc3a2e61f, 0xd78905e4, 0x5aee08ad, 0xcd461f76, 0x4021123f, 0xf102af05, 0x7c65a24c, 0xebcdb597, + 0x66aab8de, 0x72815b25, 0xffe6566c, 0x684e41b7, 0xe5294cfe, 0xf7054644, 0x7a624b0d, 0xedca5cd6, 0x60ad519f, 0x7486b264, 0xf9e1bf2d, + 0x6e49a8f6, 0xe32ea5bf, 0xfd0c7d86, 0x706b70cf, 0xe7c36714, 0x6aa46a5d, 0x7e8f89a6, 0xf3e884ef, 0x64409334, 0xe9279e7d, 0xfb0b94c7, + 0x766c998e, 0xe1c48e55, 0x6ca3831c, 0x788860e7, 0xf5ef6dae, 0x62477a75, 0xef20773c, 0xbc06940d, 0x31619944, 0xa6c98e9f, 0x2bae83d6, + 0x3f85602d, 0xb2e26d64, 0x254a7abf, 0xa82d77f6, 0xba017d4c, 0x37667005, 0xa0ce67de, 0x2da96a97, 0x3982896c, 0xb4e58425, 0x234d93fe, + 0xae2a9eb7, 0xb008468e, 0x3d6f4bc7, 0xaac75c1c, 0x27a05155, 0x338bb2ae, 0xbeecbfe7, 0x2944a83c, 0xa423a575, 0xb60fafcf, 0x3b68a286, + 0xacc0b55d, 0x21a7b814, 0x358c5bef, 0xb8eb56a6, 0x2f43417d, 0xa2244c34, 0x1307f10e, 0x9e60fc47, 0x09c8eb9c, 0x84afe6d5, 0x9084052e, + 0x1de30867, 0x8a4b1fbc, 0x072c12f5, 0x1500184f, 0x98671506, 0x0fcf02dd, 0x82a80f94, 0x9683ec6f, 0x1be4e126, 0x8c4cf6fd, 0x012bfbb4, + 0x1f09238d, 0x926e2ec4, 0x05c6391f, 0x88a13456, 0x9c8ad7ad, 0x11eddae4, 0x8645cd3f, 0x0b22c076, 0x190ecacc, 0x9469c785, 0x03c1d05e, + 0x8ea6dd17, 0x9a8d3eec, 0x17ea33a5, 0x8042247e, 0x0d252937, 0xe2055e0b, 0x6f625342, 0xf8ca4499, 0x75ad49d0, 0x6186aa2b, 0xece1a762, + 0x7b49b0b9, 0xf62ebdf0, 0xe402b74a, 0x6965ba03, 0xfecdadd8, 0x73aaa091, 0x6781436a, 0xeae64e23, 0x7d4e59f8, 0xf02954b1, 0xee0b8c88, + 0x636c81c1, 0xf4c4961a, 0x79a39b53, 0x6d8878a8, 0xe0ef75e1, 0x7747623a, 0xfa206f73, 0xe80c65c9, 0x656b6880, 0xf2c37f5b, 0x7fa47212, + 0x6b8f91e9, 0xe6e89ca0, 0x71408b7b, 0xfc278632, 0x4d043b08, 0xc0633641, 0x57cb219a, 0xdaac2cd3, 0xce87cf28, 0x43e0c261, 0xd448d5ba, + 0x592fd8f3, 0x4b03d249, 0xc664df00, 0x51ccc8db, 0xdcabc592, 0xc8802669, 0x45e72b20, 0xd24f3cfb, 0x5f2831b2, 0x410ae98b, 0xcc6de4c2, + 0x5bc5f319, 0xd6a2fe50, 0xc2891dab, 0x4fee10e2, 0xd8460739, 0x55210a70, 0x470d00ca, 0xca6a0d83, 0x5dc21a58, 0xd0a51711, 0xc48ef4ea, + 0x49e9f9a3, 0xde41ee78, 0x5326e331}, + {0x00000000, 0x780d281b, 0xf01a5036, 0x8817782d, 0xe035a06c, 0x98388877, 0x102ff05a, 0x6822d841, 0xc06b40d9, 0xb86668c2, 0x307110ef, + 0x487c38f4, 0x205ee0b5, 0x5853c8ae, 0xd044b083, 0xa8499898, 0x37ca41b6, 0x4fc769ad, 0xc7d01180, 0xbfdd399b, 0xd7ffe1da, 0xaff2c9c1, + 0x27e5b1ec, 0x5fe899f7, 0xf7a1016f, 0x8fac2974, 0x07bb5159, 0x7fb67942, 0x1794a103, 0x6f998918, 0xe78ef135, 0x9f83d92e, 0xd9894268, + 0xa1846a73, 0x2993125e, 0x519e3a45, 0x39bce204, 0x41b1ca1f, 0xc9a6b232, 0xb1ab9a29, 0x19e202b1, 0x61ef2aaa, 0xe9f85287, 0x91f57a9c, + 0xf9d7a2dd, 0x81da8ac6, 0x09cdf2eb, 0x71c0daf0, 0xee4303de, 0x964e2bc5, 0x1e5953e8, 0x66547bf3, 0x0e76a3b2, 0x767b8ba9, 0xfe6cf384, + 0x8661db9f, 0x2e284307, 0x56256b1c, 0xde321331, 0xa63f3b2a, 0xce1de36b, 0xb610cb70, 0x3e07b35d, 0x460a9b46, 0xb21385d0, 0xca1eadcb, + 0x4209d5e6, 0x3a04fdfd, 0x522625bc, 0x2a2b0da7, 0xa23c758a, 0xda315d91, 0x7278c509, 0x0a75ed12, 0x8262953f, 0xfa6fbd24, 0x924d6565, + 0xea404d7e, 0x62573553, 0x1a5a1d48, 0x85d9c466, 0xfdd4ec7d, 0x75c39450, 0x0dcebc4b, 0x65ec640a, 0x1de14c11, 0x95f6343c, 0xedfb1c27, + 0x45b284bf, 0x3dbfaca4, 0xb5a8d489, 0xcda5fc92, 0xa58724d3, 0xdd8a0cc8, 0x559d74e5, 0x2d905cfe, 0x6b9ac7b8, 0x1397efa3, 0x9b80978e, + 0xe38dbf95, 0x8baf67d4, 0xf3a24fcf, 0x7bb537e2, 0x03b81ff9, 0xabf18761, 0xd3fcaf7a, 0x5bebd757, 0x23e6ff4c, 0x4bc4270d, 0x33c90f16, + 0xbbde773b, 0xc3d35f20, 0x5c50860e, 0x245dae15, 0xac4ad638, 0xd447fe23, 0xbc652662, 0xc4680e79, 0x4c7f7654, 0x34725e4f, 0x9c3bc6d7, + 0xe436eecc, 0x6c2196e1, 0x142cbefa, 0x7c0e66bb, 0x04034ea0, 0x8c14368d, 0xf4191e96, 0xd33acba5, 0xab37e3be, 0x23209b93, 0x5b2db388, + 0x330f6bc9, 0x4b0243d2, 0xc3153bff, 0xbb1813e4, 0x13518b7c, 0x6b5ca367, 0xe34bdb4a, 0x9b46f351, 0xf3642b10, 0x8b69030b, 0x037e7b26, + 0x7b73533d, 0xe4f08a13, 0x9cfda208, 0x14eada25, 0x6ce7f23e, 0x04c52a7f, 0x7cc80264, 0xf4df7a49, 0x8cd25252, 0x249bcaca, 0x5c96e2d1, + 0xd4819afc, 0xac8cb2e7, 0xc4ae6aa6, 0xbca342bd, 0x34b43a90, 0x4cb9128b, 0x0ab389cd, 0x72bea1d6, 0xfaa9d9fb, 0x82a4f1e0, 0xea8629a1, + 0x928b01ba, 0x1a9c7997, 0x6291518c, 0xcad8c914, 0xb2d5e10f, 0x3ac29922, 0x42cfb139, 0x2aed6978, 0x52e04163, 0xdaf7394e, 0xa2fa1155, + 0x3d79c87b, 0x4574e060, 0xcd63984d, 0xb56eb056, 0xdd4c6817, 0xa541400c, 0x2d563821, 0x555b103a, 0xfd1288a2, 0x851fa0b9, 0x0d08d894, + 0x7505f08f, 0x1d2728ce, 0x652a00d5, 0xed3d78f8, 0x953050e3, 0x61294e75, 0x1924666e, 0x91331e43, 0xe93e3658, 0x811cee19, 0xf911c602, + 0x7106be2f, 0x090b9634, 0xa1420eac, 0xd94f26b7, 0x51585e9a, 0x29557681, 0x4177aec0, 0x397a86db, 0xb16dfef6, 0xc960d6ed, 0x56e30fc3, + 0x2eee27d8, 0xa6f95ff5, 0xdef477ee, 0xb6d6afaf, 0xcedb87b4, 0x46ccff99, 0x3ec1d782, 0x96884f1a, 0xee856701, 0x66921f2c, 0x1e9f3737, + 0x76bdef76, 0x0eb0c76d, 0x86a7bf40, 0xfeaa975b, 0xb8a00c1d, 0xc0ad2406, 0x48ba5c2b, 0x30b77430, 0x5895ac71, 0x2098846a, 0xa88ffc47, + 0xd082d45c, 0x78cb4cc4, 0x00c664df, 0x88d11cf2, 0xf0dc34e9, 0x98feeca8, 0xe0f3c4b3, 0x68e4bc9e, 0x10e99485, 0x8f6a4dab, 0xf76765b0, + 0x7f701d9d, 0x077d3586, 0x6f5fedc7, 0x1752c5dc, 0x9f45bdf1, 0xe74895ea, 0x4f010d72, 0x370c2569, 0xbf1b5d44, 0xc716755f, 0xaf34ad1e, + 0xd7398505, 0x5f2efd28, 0x2723d533}, + {0x00000000, 0x1168574f, 0x22d0ae9e, 0x33b8f9d1, 0xf3bd9c39, 0xe2d5cb76, 0xd16d32a7, 0xc00565e8, 0xe67b3973, 0xf7136e3c, 0xc4ab97ed, + 0xd5c3c0a2, 0x15c6a54a, 0x04aef205, 0x37160bd4, 0x267e5c9b, 0xccf772e6, 0xdd9f25a9, 0xee27dc78, 0xff4f8b37, 0x3f4aeedf, 0x2e22b990, + 0x1d9a4041, 0x0cf2170e, 0x2a8c4b95, 0x3be41cda, 0x085ce50b, 0x1934b244, 0xd931d7ac, 0xc85980e3, 0xfbe17932, 0xea892e7d, 0x2ff224c8, + 0x3e9a7387, 0x0d228a56, 0x1c4add19, 0xdc4fb8f1, 0xcd27efbe, 0xfe9f166f, 0xeff74120, 0xc9891dbb, 0xd8e14af4, 0xeb59b325, 0xfa31e46a, + 0x3a348182, 0x2b5cd6cd, 0x18e42f1c, 0x098c7853, 0xe305562e, 0xf26d0161, 0xc1d5f8b0, 0xd0bdafff, 0x10b8ca17, 0x01d09d58, 0x32686489, + 0x230033c6, 0x057e6f5d, 0x14163812, 0x27aec1c3, 0x36c6968c, 0xf6c3f364, 0xe7aba42b, 0xd4135dfa, 0xc57b0ab5, 0xe9f98894, 0xf891dfdb, + 0xcb29260a, 0xda417145, 0x1a4414ad, 0x0b2c43e2, 0x3894ba33, 0x29fced7c, 0x0f82b1e7, 0x1eeae6a8, 0x2d521f79, 0x3c3a4836, 0xfc3f2dde, + 0xed577a91, 0xdeef8340, 0xcf87d40f, 0x250efa72, 0x3466ad3d, 0x07de54ec, 0x16b603a3, 0xd6b3664b, 0xc7db3104, 0xf463c8d5, 0xe50b9f9a, + 0xc375c301, 0xd21d944e, 0xe1a56d9f, 0xf0cd3ad0, 0x30c85f38, 0x21a00877, 0x1218f1a6, 0x0370a6e9, 0xc60bac5c, 0xd763fb13, 0xe4db02c2, + 0xf5b3558d, 0x35b63065, 0x24de672a, 0x17669efb, 0x060ec9b4, 0x2070952f, 0x3118c260, 0x02a03bb1, 0x13c86cfe, 0xd3cd0916, 0xc2a55e59, + 0xf11da788, 0xe075f0c7, 0x0afcdeba, 0x1b9489f5, 0x282c7024, 0x3944276b, 0xf9414283, 0xe82915cc, 0xdb91ec1d, 0xcaf9bb52, 0xec87e7c9, + 0xfdefb086, 0xce574957, 0xdf3f1e18, 0x1f3a7bf0, 0x0e522cbf, 0x3dead56e, 0x2c828221, 0x65eed02d, 0x74868762, 0x473e7eb3, 0x565629fc, + 0x96534c14, 0x873b1b5b, 0xb483e28a, 0xa5ebb5c5, 0x8395e95e, 0x92fdbe11, 0xa14547c0, 0xb02d108f, 0x70287567, 0x61402228, 0x52f8dbf9, + 0x43908cb6, 0xa919a2cb, 0xb871f584, 0x8bc90c55, 0x9aa15b1a, 0x5aa43ef2, 0x4bcc69bd, 0x7874906c, 0x691cc723, 0x4f629bb8, 0x5e0accf7, + 0x6db23526, 0x7cda6269, 0xbcdf0781, 0xadb750ce, 0x9e0fa91f, 0x8f67fe50, 0x4a1cf4e5, 0x5b74a3aa, 0x68cc5a7b, 0x79a40d34, 0xb9a168dc, + 0xa8c93f93, 0x9b71c642, 0x8a19910d, 0xac67cd96, 0xbd0f9ad9, 0x8eb76308, 0x9fdf3447, 0x5fda51af, 0x4eb206e0, 0x7d0aff31, 0x6c62a87e, + 0x86eb8603, 0x9783d14c, 0xa43b289d, 0xb5537fd2, 0x75561a3a, 0x643e4d75, 0x5786b4a4, 0x46eee3eb, 0x6090bf70, 0x71f8e83f, 0x424011ee, + 0x532846a1, 0x932d2349, 0x82457406, 0xb1fd8dd7, 0xa095da98, 0x8c1758b9, 0x9d7f0ff6, 0xaec7f627, 0xbfafa168, 0x7faac480, 0x6ec293cf, + 0x5d7a6a1e, 0x4c123d51, 0x6a6c61ca, 0x7b043685, 0x48bccf54, 0x59d4981b, 0x99d1fdf3, 0x88b9aabc, 0xbb01536d, 0xaa690422, 0x40e02a5f, + 0x51887d10, 0x623084c1, 0x7358d38e, 0xb35db666, 0xa235e129, 0x918d18f8, 0x80e54fb7, 0xa69b132c, 0xb7f34463, 0x844bbdb2, 0x9523eafd, + 0x55268f15, 0x444ed85a, 0x77f6218b, 0x669e76c4, 0xa3e57c71, 0xb28d2b3e, 0x8135d2ef, 0x905d85a0, 0x5058e048, 0x4130b707, 0x72884ed6, + 0x63e01999, 0x459e4502, 0x54f6124d, 0x674eeb9c, 0x7626bcd3, 0xb623d93b, 0xa74b8e74, 0x94f377a5, 0x859b20ea, 0x6f120e97, 0x7e7a59d8, + 0x4dc2a009, 0x5caaf746, 0x9caf92ae, 0x8dc7c5e1, 0xbe7f3c30, 0xaf176b7f, 0x896937e4, 0x980160ab, 0xabb9997a, 0xbad1ce35, 0x7ad4abdd, + 0x6bbcfc92, 0x58040543, 0x496c520c}, + {0x00000000, 0xcadca15b, 0x94b943b7, 0x5e65e2ec, 0x9f6e466a, 0x55b2e731, 0x0bd705dd, 0xc10ba486, 0x3edd8cd4, 0xf4012d8f, 0xaa64cf63, + 0x60b86e38, 0xa1b3cabe, 0x6b6f6be5, 0x350a8909, 0xffd62852, 0xcba7d8ad, 0x017b79f6, 0x5f1e9b1a, 0x95c23a41, 0x54c99ec7, 0x9e153f9c, + 0xc070dd70, 0x0aac7c2b, 0xf57a5479, 0x3fa6f522, 0x61c317ce, 0xab1fb695, 0x6a141213, 0xa0c8b348, 0xfead51a4, 0x3471f0ff, 0x2152705f, + 0xeb8ed104, 0xb5eb33e8, 0x7f3792b3, 0xbe3c3635, 0x74e0976e, 0x2a857582, 0xe059d4d9, 0x1f8ffc8b, 0xd5535dd0, 0x8b36bf3c, 0x41ea1e67, + 0x80e1bae1, 0x4a3d1bba, 0x1458f956, 0xde84580d, 0xeaf5a8f2, 0x202909a9, 0x7e4ceb45, 0xb4904a1e, 0x759bee98, 0xbf474fc3, 0xe122ad2f, + 0x2bfe0c74, 0xd4282426, 0x1ef4857d, 0x40916791, 0x8a4dc6ca, 0x4b46624c, 0x819ac317, 0xdfff21fb, 0x152380a0, 0x42a4e0be, 0x887841e5, + 0xd61da309, 0x1cc10252, 0xddcaa6d4, 0x1716078f, 0x4973e563, 0x83af4438, 0x7c796c6a, 0xb6a5cd31, 0xe8c02fdd, 0x221c8e86, 0xe3172a00, + 0x29cb8b5b, 0x77ae69b7, 0xbd72c8ec, 0x89033813, 0x43df9948, 0x1dba7ba4, 0xd766daff, 0x166d7e79, 0xdcb1df22, 0x82d43dce, 0x48089c95, + 0xb7deb4c7, 0x7d02159c, 0x2367f770, 0xe9bb562b, 0x28b0f2ad, 0xe26c53f6, 0xbc09b11a, 0x76d51041, 0x63f690e1, 0xa92a31ba, 0xf74fd356, + 0x3d93720d, 0xfc98d68b, 0x364477d0, 0x6821953c, 0xa2fd3467, 0x5d2b1c35, 0x97f7bd6e, 0xc9925f82, 0x034efed9, 0xc2455a5f, 0x0899fb04, + 0x56fc19e8, 0x9c20b8b3, 0xa851484c, 0x628de917, 0x3ce80bfb, 0xf634aaa0, 0x373f0e26, 0xfde3af7d, 0xa3864d91, 0x695aecca, 0x968cc498, + 0x5c5065c3, 0x0235872f, 0xc8e92674, 0x09e282f2, 0xc33e23a9, 0x9d5bc145, 0x5787601e, 0x33550079, 0xf989a122, 0xa7ec43ce, 0x6d30e295, + 0xac3b4613, 0x66e7e748, 0x388205a4, 0xf25ea4ff, 0x0d888cad, 0xc7542df6, 0x9931cf1a, 0x53ed6e41, 0x92e6cac7, 0x583a6b9c, 0x065f8970, + 0xcc83282b, 0xf8f2d8d4, 0x322e798f, 0x6c4b9b63, 0xa6973a38, 0x679c9ebe, 0xad403fe5, 0xf325dd09, 0x39f97c52, 0xc62f5400, 0x0cf3f55b, + 0x529617b7, 0x984ab6ec, 0x5941126a, 0x939db331, 0xcdf851dd, 0x0724f086, 0x12077026, 0xd8dbd17d, 0x86be3391, 0x4c6292ca, 0x8d69364c, + 0x47b59717, 0x19d075fb, 0xd30cd4a0, 0x2cdafcf2, 0xe6065da9, 0xb863bf45, 0x72bf1e1e, 0xb3b4ba98, 0x79681bc3, 0x270df92f, 0xedd15874, + 0xd9a0a88b, 0x137c09d0, 0x4d19eb3c, 0x87c54a67, 0x46ceeee1, 0x8c124fba, 0xd277ad56, 0x18ab0c0d, 0xe77d245f, 0x2da18504, 0x73c467e8, + 0xb918c6b3, 0x78136235, 0xb2cfc36e, 0xecaa2182, 0x267680d9, 0x71f1e0c7, 0xbb2d419c, 0xe548a370, 0x2f94022b, 0xee9fa6ad, 0x244307f6, + 0x7a26e51a, 0xb0fa4441, 0x4f2c6c13, 0x85f0cd48, 0xdb952fa4, 0x11498eff, 0xd0422a79, 0x1a9e8b22, 0x44fb69ce, 0x8e27c895, 0xba56386a, + 0x708a9931, 0x2eef7bdd, 0xe433da86, 0x25387e00, 0xefe4df5b, 0xb1813db7, 0x7b5d9cec, 0x848bb4be, 0x4e5715e5, 0x1032f709, 0xdaee5652, + 0x1be5f2d4, 0xd139538f, 0x8f5cb163, 0x45801038, 0x50a39098, 0x9a7f31c3, 0xc41ad32f, 0x0ec67274, 0xcfcdd6f2, 0x051177a9, 0x5b749545, + 0x91a8341e, 0x6e7e1c4c, 0xa4a2bd17, 0xfac75ffb, 0x301bfea0, 0xf1105a26, 0x3bccfb7d, 0x65a91991, 0xaf75b8ca, 0x9b044835, 0x51d8e96e, + 0x0fbd0b82, 0xc561aad9, 0x046a0e5f, 0xceb6af04, 0x90d34de8, 0x5a0fecb3, 0xa5d9c4e1, 0x6f0565ba, 0x31608756, 0xfbbc260d, 0x3ab7828b, + 0xf06b23d0, 0xae0ec13c, 0x64d26067}}; + +static inline uint32_t +StrCrc(const char* Data) +{ + uint32_t CRC = 0xFFFFFFFF; + while (*Data) + { + char16_t C = *Data++; + int32_t CL = (C & 255); + CRC = (CRC << 8) ^ CRCTable[(CRC >> 24) ^ CL]; + int32_t CH = (C >> 8) & 255; + CRC = (CRC << 8) ^ CRCTable[(CRC >> 24) ^ CH]; + } + return ~CRC; +} + +#define BYTESWAP_ORDER32(x) (((x) >> 24) + (((x) >> 8) & 0xff00) + (((x) << 8) & 0xff0000) + ((x) << 24)) +#define UE_PTRDIFF_TO_INT32(argument) static_cast<int32_t>(argument) + +template<typename T> +constexpr T +Align(T Val, uint64_t Alignment) +{ + return (T)(((uint64_t)Val + Alignment - 1) & ~(Alignment - 1)); +} + +static uint32_t +MemCRC(const void* InData, size_t Length, uint32_t CRC = 0) +{ + // Based on the Slicing-by-8 implementation found here: + // http://slicing-by-8.sourceforge.net/ + + CRC = ~BYTESWAP_ORDER32(CRC); + + const uint8_t* __restrict Data = (uint8_t*)InData; + + // First we need to align to 32-bits + int32_t InitBytes = UE_PTRDIFF_TO_INT32(Align(Data, 4) - Data); + + if (Length > InitBytes) + { + Length -= InitBytes; + + for (; InitBytes; --InitBytes) + { + CRC = (CRC >> 8) ^ CRCTablesSB8[0][(CRC & 0xFF) ^ *Data++]; + } + + auto Data4 = (const uint32_t*)Data; + for (size_t Repeat = Length / 8; Repeat; --Repeat) + { + uint32_t V1 = *Data4++ ^ CRC; + uint32_t V2 = *Data4++; + CRC = CRCTablesSB8[7][V1 & 0xFF] ^ CRCTablesSB8[6][(V1 >> 8) & 0xFF] ^ CRCTablesSB8[5][(V1 >> 16) & 0xFF] ^ + CRCTablesSB8[4][V1 >> 24] ^ CRCTablesSB8[3][V2 & 0xFF] ^ CRCTablesSB8[2][(V2 >> 8) & 0xFF] ^ + CRCTablesSB8[1][(V2 >> 16) & 0xFF] ^ CRCTablesSB8[0][V2 >> 24]; + } + Data = (const uint8_t*)Data4; + + Length %= 8; + } + + for (; Length; --Length) + { + CRC = (CRC >> 8) ^ CRCTablesSB8[0][(CRC & 0xFF) ^ *Data++]; + } + + return BYTESWAP_ORDER32(~CRC); +} + +struct CorruptionTrailer +{ + enum + { + /** Arbitrary number used to identify corruption **/ + MagicConstant = 0x1e873d89 + }; + + uint32_t Magic = MagicConstant; + uint32_t Version = 1; + uint32_t CRCofPayload = 0; + uint32_t SizeOfPayload = 0; + + void Initialize(const void* Data, size_t Size) + { + CRCofPayload = MemCRC(Data, Size); + SizeOfPayload = (uint32_t)Size; + } +}; + +std::wstring +GenerateDdcPath(std::string_view Key, std::filesystem::path& rootDir) +{ + std::filesystem::path FilePath = rootDir; + + std::string k8{Key}; + for (auto& c : k8) + c = (char)toupper(c); + + const uint32_t Hash = StrCrc(k8.c_str()); + + std::wstring DirName; + + DirName = u'0' + ((Hash / 100) % 10); + FilePath /= DirName; + DirName = u'0' + ((Hash / 10) % 10); + FilePath /= DirName; + DirName = u'0' + (Hash % 10); + FilePath /= DirName; + + FilePath /= Key; + + auto NativePath = FilePath.native(); + NativePath.append(L".udd"); + + return NativePath; +} + +} // namespace UE + +////////////////////////////////////////////////////////////////////////// + +FileCacheStore::FileCacheStore(const char* RootDir, const char* ReadRootDir) +{ + // Ensure root directory exists - create if it doesn't exist already + + spdlog::info("Initializing FileCacheStore at '{}'", std::string_view(RootDir)); + + m_RootDir = RootDir; + + std::error_code ErrorCode; + + std::filesystem::create_directories(m_RootDir, ErrorCode); + + if (ErrorCode) + { + ExtendableStringBuilder<256> Name; + WideToUtf8(m_RootDir.c_str(), Name); + + spdlog::error("Could not open file cache directory '{}' for writing ({})", Name.c_str(), ErrorCode.message()); + + m_IsOk = false; + } + + if (ReadRootDir) + { + m_ReadRootDir = ReadRootDir; + + if (std::filesystem::exists(m_ReadRootDir, ErrorCode)) + { + spdlog::info("FileCacheStore will use additional read tree at '{}'", std::string_view(ReadRootDir)); + + m_ReadRootIsValid = true; + } + } +} + +FileCacheStore::~FileCacheStore() +{ +} + +bool +FileCacheStore::Get(std::string_view Key, CacheValue& OutValue) +{ + CAtlFile File; + + std::wstring nativePath; + + HRESULT hRes = E_FAIL; + + if (m_ReadRootDir.empty() == false) + { + nativePath = UE::GenerateDdcPath(Key, m_ReadRootDir); + + hRes = File.Create(nativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); + } + + if (FAILED(hRes)) + { + nativePath = UE::GenerateDdcPath(Key, m_RootDir); + + hRes = File.Create(nativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); + } + + if (FAILED(hRes)) + { + spdlog::debug("GET MISS {}", Key); + + return false; + } + + ULONGLONG FileSize; + File.GetSize(FileSize); + + if (FileSize <= 16) + { + return false; + } + + FileSize -= 16; // CorruptionWrapper trailer + + IoBuffer Value(FileSize); + + uint8_t* ReadPointer = (uint8_t*)Value.Data(); + + while (FileSize) + { + const int MaxChunkSize = 16 * 1024 * 1024; + const int ChunkSize = gsl::narrow_cast<int>((FileSize > MaxChunkSize) ? MaxChunkSize : FileSize); + + DWORD BytesRead = 0; + hRes = File.Read(ReadPointer, ChunkSize, BytesRead); + + if (FAILED(hRes)) + { + return false; + } + + ReadPointer += BytesRead; + FileSize -= BytesRead; + } + + OutValue.Value = std::move(Value); + + spdlog::debug("GET HIT {}", Key); + + return true; +} + +void +FileCacheStore::Put(std::string_view Key, const CacheValue& Value) +{ + const void* Data = Value.Value.Data(); + size_t Size = Value.Value.Size(); + + UE::CorruptionTrailer Trailer; + Trailer.Initialize(Data, Size); + + std::wstring nativePath = UE::GenerateDdcPath(Key, m_RootDir); + + CAtlTemporaryFile File; + + spdlog::debug("PUT {}", Key); + + HRESULT hRes = File.Create(m_RootDir.c_str()); + + if (SUCCEEDED(hRes)) + { + const uint8_t* WritePointer = (const uint8_t*)Data; + + while (Size) + { + const int MaxChunkSize = 16 * 1024 * 1024; + const int ChunkSize = (int)((Size > MaxChunkSize) ? MaxChunkSize : Size); + + DWORD BytesWritten = 0; + File.Write(WritePointer, ChunkSize, &BytesWritten); + + Size -= BytesWritten; + WritePointer += BytesWritten; + } + + File.Write(&Trailer, sizeof Trailer); + hRes = File.Close(nativePath.c_str()); // This renames the file to its final name + + if (FAILED(hRes)) + { + spdlog::warn("Failed to rename temp file for key '{}' - deleting temporary file", Key); + + if (!DeleteFile(File.TempFileName())) + { + spdlog::warn("Temp file for key '{}' could not be deleted - no value persisted", Key); + } + } + } +} + +////////////////////////////////////////////////////////////////////////// + +MemoryCacheStore::MemoryCacheStore() +{ +} + +MemoryCacheStore::~MemoryCacheStore() +{ +} + +bool +MemoryCacheStore::Get(std::string_view InKey, CacheValue& OutValue) +{ + RwLock::SharedLockScope _(m_Lock); + + auto it = m_CacheMap.find(std::string(InKey)); + + if (it == m_CacheMap.end()) + { + return false; + } + else + { + OutValue.Value = it->second; + + return true; + } +} + +void +MemoryCacheStore::Put(std::string_view Key, const CacheValue& Value) +{ + RwLock::ExclusiveLockScope _(m_Lock); + m_CacheMap[std::string(Key)] = Value.Value; +} + +////////////////////////////////////////////////////////////////////////// + +ZenCacheStore::ZenCacheStore(zen::CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir} +{ + zen::CreateDirectories(RootDir); +} + +ZenCacheStore::~ZenCacheStore() +{ +} + +bool +ZenCacheStore::Get(std::string_view InBucket, const zen::IoHash& HashKey, CacheValue& OutValue) +{ + bool Ok = m_MemLayer.Get(InBucket, HashKey, OutValue); + + if (!Ok) + { + Ok = m_DiskLayer.Get(InBucket, HashKey, OutValue); + +#if 0 // This would keep file handles open + if (ok) + { + m_memLayer.Put(InBucket, HashKey, OutValue); + } +#endif + } + + return Ok; +} + +void +ZenCacheStore::Put(std::string_view InBucket, const zen::IoHash& HashKey, const CacheValue& Value) +{ + m_MemLayer.Put(InBucket, HashKey, Value); + m_DiskLayer.Put(InBucket, HashKey, Value); +} + +////////////////////////////////////////////////////////////////////////// + +ZenCacheMemoryLayer::ZenCacheMemoryLayer() +{ +} + +ZenCacheMemoryLayer::~ZenCacheMemoryLayer() +{ +} + +bool +ZenCacheMemoryLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, CacheValue& OutValue) +{ + CacheBucket* Bucket = nullptr; + + { + RwLock::SharedLockScope _(m_Lock); + + auto it = m_Buckets.find(std::string(InBucket)); + + if (it != m_Buckets.end()) + { + Bucket = &it->second; + } + } + + if (Bucket == nullptr) + return false; + + ZEN_ASSERT(Bucket != nullptr); + + return Bucket->Get(HashKey, OutValue); +} + +void +ZenCacheMemoryLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, const CacheValue& Value) +{ + CacheBucket* Bucket = nullptr; + + { + RwLock::SharedLockScope _(m_Lock); + + auto it = m_Buckets.find(std::string(InBucket)); + + if (it != m_Buckets.end()) + { + Bucket = &it->second; + } + } + + if (Bucket == nullptr) + { + // New bucket + + RwLock::ExclusiveLockScope _(m_Lock); + + Bucket = &m_Buckets[std::string(InBucket)]; + } + + ZEN_ASSERT(Bucket != nullptr); + + Bucket->Put(HashKey, Value); +} + +bool +ZenCacheMemoryLayer::CacheBucket::Get(const zen::IoHash& HashKey, CacheValue& OutValue) +{ + RwLock::SharedLockScope _(m_bucketLock); + + auto bucketIt = m_cacheMap.find(HashKey); + + if (bucketIt == m_cacheMap.end()) + { + return false; + } + + OutValue.Value = bucketIt->second; + + return true; +} + +void +ZenCacheMemoryLayer::CacheBucket::Put(const zen::IoHash& HashKey, const CacheValue& Value) +{ + RwLock::ExclusiveLockScope _(m_bucketLock); + + m_cacheMap[HashKey] = Value.Value; +} + +////////////////////////////////////////////////////////////////////////// + +class ZenFile +{ +public: + void Open(std::filesystem::path FileName, bool IsCreate); + void Read(void* Data, uint64_t Size, uint64_t Offset); + void Write(const void* Data, uint64_t Size, uint64_t Offset); + void Flush(); + void* Handle() { return m_File; } + +private: + CAtlFile m_File; +}; + +void +ZenFile::Open(std::filesystem::path FileName, bool isCreate) +{ + const DWORD dwCreationDisposition = isCreate ? CREATE_ALWAYS : OPEN_EXISTING; + + HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition); + + if (FAILED(hRes)) + { + throw std::system_error(GetLastError(), std::system_category(), "Failed to open bucket sobs file"); + } +} + +void +ZenFile::Read(void* Data, uint64_t Size, uint64_t Offset) +{ + OVERLAPPED Ovl{}; + + Ovl.Offset = DWORD(Offset & 0xffff'ffffu); + Ovl.OffsetHigh = DWORD(Offset >> 32); + + HRESULT hRes = m_File.Read(Data, gsl::narrow<DWORD>(Size), &Ovl); + + if (FAILED(hRes)) + { + throw std::system_error(GetLastError(), std::system_category(), "Failed to read from file" /* TODO: add context */); + } +} + +void +ZenFile::Write(const void* Data, uint64_t Size, uint64_t Offset) +{ + OVERLAPPED Ovl{}; + + Ovl.Offset = DWORD(Offset & 0xffff'ffffu); + Ovl.OffsetHigh = DWORD(Offset >> 32); + + HRESULT hRes = m_File.Write(Data, gsl::narrow<DWORD>(Size), &Ovl); + + if (FAILED(hRes)) + { + throw std::system_error(GetLastError(), std::system_category(), "Failed to write to file" /* TODO: add context */); + } +} + +void +ZenFile::Flush() +{ + m_File.Flush(); +} + +////////////////////////////////////////////////////////////////////////// + +class ZenLogFile +{ +public: + ZenLogFile(); + ~ZenLogFile(); + + void Open(std::filesystem::path FileName, size_t RecordSize, bool isCreate); + void Append(const void* DataPointer, uint64_t DataSize); + void Replay(std::function<void(const void*)>&& Handler); + void Flush(); + +private: + CAtlFile m_File; + size_t m_RecordSize = 1; + uint64_t m_AppendOffset = 0; +}; + +ZenLogFile::ZenLogFile() +{ +} + +ZenLogFile::~ZenLogFile() +{ +} + +void +ZenLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreate) +{ + m_RecordSize = RecordSize; + + const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; + + HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition); + + if (FAILED(hRes)) + { + throw std::system_error(GetLastError(), std::system_category(), "Failed to open log file" /* TODO: add path */); + } + + // TODO: write/validate header and log contents and prepare for appending/replay +} + +void +ZenLogFile::Replay(std::function<void(const void*)>&& Handler) +{ + std::vector<uint8_t> ReadBuffer; + + ULONGLONG LogFileSize; + m_File.GetSize(LogFileSize); + + // Ensure we end up on a clean boundary + LogFileSize -= LogFileSize % m_RecordSize; + + ReadBuffer.resize(LogFileSize); + + HRESULT hRes = m_File.Read(ReadBuffer.data(), gsl::narrow<DWORD>(ReadBuffer.size())); + + if (FAILED(hRes)) + { + throw std::system_error(GetLastError(), std::system_category(), "Failed to read log file" /* TODO: add context */); + } + + const size_t EntryCount = LogFileSize / m_RecordSize; + + for (int i = 0; i < EntryCount; ++i) + { + Handler(ReadBuffer.data() + (i * m_RecordSize)); + } +} + +void +ZenLogFile::Append(const void* DataPointer, uint64_t DataSize) +{ + HRESULT hRes = m_File.Write(DataPointer, gsl::narrow<DWORD>(DataSize)); + + if (FAILED(hRes)) + { + throw std::system_error(GetLastError(), std::system_category(), "Failed to write to log file" /* TODO: add context */); + } +} + +void +ZenLogFile::Flush() +{ +} + +template<typename T> +class TZenLogFile : public ZenLogFile +{ +public: + void Replay(std::function<void(const T&)>&& Handler) + { + ZenLogFile::Replay([&](const void* VoidPtr) { + const T& Record = *reinterpret_cast<const T*>(VoidPtr); + + Handler(Record); + }); + } + + void Append(const T& Record) { ZenLogFile::Append(&Record, sizeof Record); } +}; + +////////////////////////////////////////////////////////////////////////// + +#pragma pack(push) +#pragma pack(1) + +struct DiskLocation +{ + uint64_t Offset; + uint32_t Size; +}; + +struct DiskIndexEntry +{ + zen::IoHash Key; + DiskLocation Location; +}; + +#pragma pack(pop) + +static_assert(sizeof(DiskIndexEntry) == 32); + +struct ZenCacheDiskLayer::CacheBucket +{ + CacheBucket(CasStore& Cas); + ~CacheBucket(); + + void OpenOrCreate(std::filesystem::path BucketDir); + + bool Get(const zen::IoHash& HashKey, CacheValue& OutValue); + void Put(const zen::IoHash& HashKey, const CacheValue& Value); + void Flush(); + + void PutLargeObject(const zen::IoHash& HashKey, const CacheValue& Value); + + inline bool IsOk() const { return m_Ok; } + + CasStore& m_CasStore; + std::filesystem::path m_BucketDir; + Oid m_BucketId; + bool m_Ok = false; + uint64_t m_LargeObjectThreshold = 1024; + + ZenFile m_SobsFile; + TZenLogFile<DiskIndexEntry> m_SlogFile; + ZenFile m_SidxFile; + + void BuildPath(zen::WideStringBuilderBase& Path, const zen::IoHash& HashKey); + + RwLock m_IndexLock; + std::unordered_map<zen::IoHash, DiskLocation, zen::IoHash::Hasher> m_Index; + uint64_t m_WriteCursor = 0; +}; + +ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas) +{ +} + +ZenCacheDiskLayer::CacheBucket::~CacheBucket() +{ +} + +void +ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir) +{ + std::filesystem::create_directories(BucketDir); + + m_BucketDir = BucketDir; + + std::wstring ManifestPath{(m_BucketDir / "zen_manifest").c_str()}; + std::wstring SobsPath{(m_BucketDir / "zen.sobs").c_str()}; + std::wstring SlogPath{(m_BucketDir / "zen.slog").c_str()}; + std::wstring SidxPath{(m_BucketDir / "zen.sidx").c_str()}; + + CAtlFile ManifestFile; + + // Try opening existing file first + + bool IsNew = false; + + HRESULT hRes = ManifestFile.Create(ManifestPath.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, OPEN_EXISTING); + + if (SUCCEEDED(hRes)) + { + ULONGLONG FileSize; + ManifestFile.GetSize(FileSize); + + if (FileSize == sizeof(Oid)) + { + hRes = ManifestFile.Read(&m_BucketId, sizeof(Oid)); + + if (SUCCEEDED(hRes)) + { + m_Ok = true; + } + } + + if (!m_Ok) + ManifestFile.Close(); + } + + if (!m_Ok) + { + // This is a new bucket + + hRes = ManifestFile.Create(ManifestPath.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); + + if (FAILED(hRes)) + { + throw std::system_error(GetLastError(), std::system_category(), "Failed to create bucket manifest"); + } + + m_BucketId.Generate(); + + hRes = ManifestFile.Write(&m_BucketId, sizeof(Oid)); + + IsNew = true; + } + + // Initialize small object storage related files + + m_SobsFile.Open(SobsPath, IsNew); + m_SidxFile.Open(SidxPath, IsNew); + + // Open and replay log + + m_SlogFile.Open(SlogPath, sizeof(DiskIndexEntry), IsNew); + + uint64_t maxFileOffset = 0; + + { + // This is not technically necessary but may help future static analysis + zen::RwLock::ExclusiveLockScope _(m_IndexLock); + + m_SlogFile.Replay([&](const DiskIndexEntry& Record) { + m_Index[Record.Key] = Record.Location; + + maxFileOffset = std::max<uint64_t>(maxFileOffset, Record.Location.Offset + Record.Location.Size); + }); + } + + m_WriteCursor = (maxFileOffset + 15) & ~15; + + m_Ok = true; +} + +void +ZenCacheDiskLayer::CacheBucket::BuildPath(zen::WideStringBuilderBase& Path, const zen::IoHash& HashKey) +{ + char hex[sizeof(HashKey.Hash) * 2]; + ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, hex); + + Path.Append(m_BucketDir.c_str()); + Path.Append(L"/"); + Path.AppendAsciiRange(hex, hex + sizeof(hex)); +} + +bool +ZenCacheDiskLayer::CacheBucket::Get(const zen::IoHash& HashKey, CacheValue& OutValue) +{ + if (!m_Ok) + return false; + + { + zen::RwLock::SharedLockScope _(m_IndexLock); + + auto it = m_Index.find(HashKey); + + if (it != m_Index.end()) + { + OutValue.Value = IoBufferBuilder::MakeFromFileHandle(m_SobsFile.Handle(), it->second.Offset, it->second.Size); + + return true; + } + } + + WideStringBuilder<128> dataFilePath; + BuildPath(dataFilePath, HashKey); + + zen::IoBuffer data = IoBufferBuilder::MakeFromFile(dataFilePath.c_str()); + + if (!data) + { + return false; + } + + OutValue.Value = data; + + // TODO: should populate index? + + return true; +} + +void +ZenCacheDiskLayer::CacheBucket::Put(const zen::IoHash& HashKey, const CacheValue& Value) +{ + if (!m_Ok) + return; + + if (Value.Value.Size() >= m_LargeObjectThreshold) + PutLargeObject(HashKey, Value); + + // Small object put + + zen::RwLock::ExclusiveLockScope _(m_IndexLock); + + auto it = m_Index.find(HashKey); + + DiskLocation loc{.Offset = m_WriteCursor, .Size = gsl::narrow<uint32_t>(Value.Value.Size())}; + + m_WriteCursor = (m_WriteCursor + loc.Size + 15) & ~15; + + if (it == m_Index.end()) + { + m_Index.insert({HashKey, loc}); + } + else + { + // TODO: should check if write is idempotent and bail out if it is? + + it->second = loc; + } + + DiskIndexEntry indexEntry{.Key = HashKey, .Location = loc}; + + m_SlogFile.Append(indexEntry); + + m_SobsFile.Write(Value.Value.Data(), loc.Size, loc.Offset); + + return; +} + +void +ZenCacheDiskLayer::CacheBucket::Flush() +{ + m_SobsFile.Flush(); + m_SidxFile.Flush(); + m_SlogFile.Flush(); +} + +void +ZenCacheDiskLayer::CacheBucket::PutLargeObject(const zen::IoHash& HashKey, const CacheValue& Value) +{ + zen::WideStringBuilder<128> dataFilePath; + + BuildPath(dataFilePath, HashKey); + + CAtlTemporaryFile dataFile; + + HRESULT hRes = dataFile.Create(m_BucketDir.c_str()); + + hRes = dataFile.Write(Value.Value.Data(), gsl::narrow<DWORD>(Value.Value.Size())); + + if (FAILED(hRes)) + { + // TODO: report error! and delete temp file + + return; + } + + hRes = dataFile.Close(dataFilePath.c_str()); +} + +////////////////////////////////////////////////////////////////////////// + +ZenCacheDiskLayer::ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir) : m_RootDir(RootDir), m_CasStore(Cas) +{ +} + +ZenCacheDiskLayer::~ZenCacheDiskLayer() = default; + +bool +ZenCacheDiskLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, CacheValue& OutValue) +{ + CacheBucket* Bucket = nullptr; + + { + zen::RwLock::SharedLockScope _(m_Lock); + + auto it = m_Buckets.find(std::string(InBucket)); + + if (it != m_Buckets.end()) + { + Bucket = &it->second; + } + } + + if (Bucket == nullptr) + { + // Bucket needs to be opened/created + + zen::RwLock::ExclusiveLockScope _(m_Lock); + + auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore); + Bucket = &It.first->second; + + std::filesystem::path BucketPath = m_RootDir; + BucketPath /= std::string(InBucket); + + Bucket->OpenOrCreate(BucketPath.c_str()); + } + + ZEN_ASSERT(Bucket != nullptr); + + return Bucket->Get(HashKey, OutValue); +} + +void +ZenCacheDiskLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, const CacheValue& Value) +{ + CacheBucket* Bucket = nullptr; + + { + zen::RwLock::SharedLockScope _(m_Lock); + + auto it = m_Buckets.find(std::string(InBucket)); + + if (it != m_Buckets.end()) + { + Bucket = &it->second; + } + } + + if (Bucket == nullptr) + { + // New bucket needs to be created + + zen::RwLock::ExclusiveLockScope _(m_Lock); + + auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore); + Bucket = &It.first->second; + + std::filesystem::path bucketPath = m_RootDir; + bucketPath /= std::string(InBucket); + + Bucket->OpenOrCreate(bucketPath.c_str()); + } + + ZEN_ASSERT(Bucket != nullptr); + + if (Bucket->IsOk()) + { + Bucket->Put(HashKey, Value); + } +} + +void +ZenCacheDiskLayer::Flush() +{ + std::vector<CacheBucket*> Buckets; + Buckets.reserve(m_Buckets.size()); + + { + zen::RwLock::SharedLockScope _(m_Lock); + + for (auto& Kv : m_Buckets) + { + Buckets.push_back(&Kv.second); + } + } + + for (auto& Bucket : Buckets) + { + Bucket->Flush(); + } +} + +////////////////////////////////////////////////////////////////////////// + +ZenCacheTracker::ZenCacheTracker(ZenCacheStore& CacheStore) +{ + ZEN_UNUSED(CacheStore); +} + +ZenCacheTracker::~ZenCacheTracker() +{ +} + +void +ZenCacheTracker::TrackAccess(std::string_view Bucket, const zen::IoHash& HashKey) +{ + ZEN_UNUSED(Bucket); + ZEN_UNUSED(HashKey); +} diff --git a/zenserver/cache/cachestore.h b/zenserver/cache/cachestore.h new file mode 100644 index 000000000..1ac01279b --- /dev/null +++ b/zenserver/cache/cachestore.h @@ -0,0 +1,175 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/IoBuffer.h> +#include <zencore/iohash.h> +#include <zencore/thread.h> +#include <zencore/uid.h> +#include <zenstore/cas.h> +#include <compare> +#include <filesystem> +#include <unordered_map> + +namespace zen { + +class WideStringBuilderBase; +class CasStore; + +} // namespace zen + +struct CacheValue +{ + zen::IoBuffer Value; +}; + +/****************************************************************************** + + /$$ /$$/$$ /$$ /$$$$$$ /$$ + | $$ /$$| $$ | $$ /$$__ $$ | $$ + | $$ /$$/| $$ | $$ | $$ \__/ /$$$$$$ /$$$$$$| $$$$$$$ /$$$$$$ + | $$$$$/ | $$ / $$/ | $$ |____ $$/$$_____| $$__ $$/$$__ $$ + | $$ $$ \ $$ $$/ | $$ /$$$$$$| $$ | $$ \ $| $$$$$$$$ + | $$\ $$ \ $$$/ | $$ $$/$$__ $| $$ | $$ | $| $$_____/ + | $$ \ $$ \ $/ | $$$$$$| $$$$$$| $$$$$$| $$ | $| $$$$$$$ + |__/ \__/ \_/ \______/ \_______/\_______|__/ |__/\_______/ + + Basic Key-Value cache. No restrictions on keys, and values are always opaque + binary blobs. + +******************************************************************************/ + +class CacheStore +{ +public: + virtual bool Get(std::string_view Key, CacheValue& OutValue) = 0; + virtual void Put(std::string_view Key, const CacheValue& Value) = 0; +}; + +/** File system based implementation + + Emulates the behaviour of UE4 with regards to file system structure, + and also adds a file corruption trailer to remain compatible with + the file-system based implementation (this should be made configurable) + + */ +class FileCacheStore : public CacheStore +{ +public: + FileCacheStore(const char* RootDir, const char* ReadRootDir = nullptr); + ~FileCacheStore(); + + virtual bool Get(std::string_view Key, CacheValue& OutValue) override; + virtual void Put(std::string_view Key, const CacheValue& Value) override; + +private: + std::filesystem::path m_RootDir; + std::filesystem::path m_ReadRootDir; + bool m_IsOk = true; + bool m_ReadRootIsValid = false; +}; + +class MemoryCacheStore : public CacheStore +{ +public: + MemoryCacheStore(); + ~MemoryCacheStore(); + + virtual bool Get(std::string_view Key, CacheValue& OutValue) override; + virtual void Put(std::string_view Key, const CacheValue& Value) override; + +private: + zen::RwLock m_Lock; + std::unordered_map<std::string, zen::IoBuffer> m_CacheMap; +}; + +/****************************************************************************** + + /$$$$$$$$ /$$$$$$ /$$ + |_____ $$ /$$__ $$ | $$ + /$$/ /$$$$$$ /$$$$$$$ | $$ \__/ /$$$$$$ /$$$$$$| $$$$$$$ /$$$$$$ + /$$/ /$$__ $| $$__ $$ | $$ |____ $$/$$_____| $$__ $$/$$__ $$ + /$$/ | $$$$$$$| $$ \ $$ | $$ /$$$$$$| $$ | $$ \ $| $$$$$$$$ + /$$/ | $$_____| $$ | $$ | $$ $$/$$__ $| $$ | $$ | $| $$_____/ + /$$$$$$$| $$$$$$| $$ | $$ | $$$$$$| $$$$$$| $$$$$$| $$ | $| $$$$$$$ + |________/\_______|__/ |__/ \______/ \_______/\_______|__/ |__/\_______/ + + Cache store for UE5. Restricts keys to "{bucket}/{hash}" pairs where the hash + is 40 (hex) chars in size. Values may be opaque blobs or structured objects + which can in turn contain references to other objects. + +******************************************************************************/ + +class ZenCacheMemoryLayer +{ +public: + ZenCacheMemoryLayer(); + ~ZenCacheMemoryLayer(); + + bool Get(std::string_view Bucket, const zen::IoHash& HashKey, CacheValue& OutValue); + void Put(std::string_view Bucket, const zen::IoHash& HashKey, const CacheValue& Value); + +private: + struct CacheBucket + { + zen::RwLock m_bucketLock; + std::unordered_map<zen::IoHash, zen::IoBuffer, zen::IoHash::Hasher> m_cacheMap; + + bool Get(const zen::IoHash& HashKey, CacheValue& OutValue); + void Put(const zen::IoHash& HashKey, const CacheValue& Value); + }; + + zen::RwLock m_Lock; + std::unordered_map<std::string, CacheBucket> m_Buckets; +}; + +class ZenCacheDiskLayer +{ +public: + ZenCacheDiskLayer(zen::CasStore& Cas, const std::filesystem::path& RootDir); + ~ZenCacheDiskLayer(); + + bool Get(std::string_view Bucket, const zen::IoHash& HashKey, CacheValue& OutValue); + void Put(std::string_view Bucket, const zen::IoHash& HashKey, const CacheValue& Value); + + void Flush(); + +private: + /** A cache bucket manages a single directory containing + metadata and data for that bucket + */ + struct CacheBucket; + + zen::CasStore& m_CasStore; + std::filesystem::path m_RootDir; + zen::RwLock m_Lock; + std::unordered_map<std::string, CacheBucket> m_Buckets; // TODO: make this case insensitive +}; + +class ZenCacheStore +{ +public: + ZenCacheStore(zen::CasStore& Cas, const std::filesystem::path& RootDir); + ~ZenCacheStore(); + + virtual bool Get(std::string_view Bucket, const zen::IoHash& HashKey, CacheValue& OutValue); + virtual void Put(std::string_view Bucket, const zen::IoHash& HashKey, const CacheValue& Value); + +private: + std::filesystem::path m_RootDir; + ZenCacheMemoryLayer m_MemLayer; + ZenCacheDiskLayer m_DiskLayer; +}; + +/** Tracks cache entry access, stats and orchestrates cleanup activities + */ +class ZenCacheTracker +{ +public: + ZenCacheTracker(ZenCacheStore& CacheStore); + ~ZenCacheTracker(); + + void TrackAccess(std::string_view Bucket, const zen::IoHash& HashKey); + +private: +}; diff --git a/zenserver/cache/kvcache.cpp b/zenserver/cache/kvcache.cpp new file mode 100644 index 000000000..404b17e5a --- /dev/null +++ b/zenserver/cache/kvcache.cpp @@ -0,0 +1,208 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "kvcache.h" + +#include <zencore/httpserver.h> +#include <zencore/memory.h> +#include <zencore/timer.h> +#include "cachestore.h" +#include "upstream/jupiter.h" + +#include <rocksdb/db.h> +#include <spdlog/spdlog.h> + +namespace zen { + +namespace rocksdb = ROCKSDB_NAMESPACE; +using namespace fmt::literals; +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +struct HttpKvCacheService::AccessTracker +{ + AccessTracker(); + ~AccessTracker(); + + void TrackAccess(std::string_view Key); + void Flush(); + +private: + RwLock m_Lock; + ChunkingLinearAllocator m_AccessRecordAllocator{8192}; +}; + +HttpKvCacheService::AccessTracker::AccessTracker() +{ +} + +HttpKvCacheService::AccessTracker::~AccessTracker() +{ + RwLock::ExclusiveLockScope _(m_Lock); +} + +void +HttpKvCacheService::AccessTracker::Flush() +{ + RwLock::ExclusiveLockScope _(m_Lock); + + m_AccessRecordAllocator.Reset(); +} + +void +HttpKvCacheService::AccessTracker::TrackAccess(std::string_view Key) +{ + // Once it matters, this should use a thread-local means of updating this data, + // like Concurrency::combinable or similar + + RwLock::ExclusiveLockScope _(m_Lock); + + const uint64_t KeySize = Key.size(); + void* Ptr = m_AccessRecordAllocator.Alloc(KeySize + 1); + memcpy(Ptr, Key.data(), KeySize); + reinterpret_cast<uint8_t*>(Ptr)[KeySize] = 0; +} + +////////////////////////////////////////////////////////////////////////// + +HttpKvCacheService::HttpKvCacheService() +{ + m_Cloud = new CloudCacheClient("https://jupiter.devtools.epicgames.com"sv, + "ue4.ddc"sv /* namespace */, + "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv /* provider */, + "0oao91lrhqPiAlaGD0x7"sv /* client id */, + "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv /* oauth secret */); + + m_AccessTracker = std::make_unique<AccessTracker>(); +} + +HttpKvCacheService::~HttpKvCacheService() +{ +} + +const char* +HttpKvCacheService::BaseUri() const +{ + return "/cache/"; +} + +void +HttpKvCacheService::HandleRequest(zen::HttpServerRequest& Request) +{ + using namespace std::literals; + + std::string_view Key = Request.RelativeUri(); + + switch (auto Verb = Request.RequestVerb()) + { + using enum zen::HttpVerb; + + case kHead: + case kGet: + { + m_AccessTracker->TrackAccess(Key); + + CacheValue Value; + bool Success = m_cache.Get(Key, Value); + + if (!Success) + { + // Success = m_cache_.Get(Key, Value); + + if (!Success) + { + CloudCacheSession Session(m_Cloud); + + zen::Stopwatch Timer; + + if (IoBuffer CloudValue = Session.Get("default", Key)) + { + Success = true; + + spdlog::debug("upstream HIT after {:5} {:6}! {}", + zen::NiceTimeSpanMs(Timer.getElapsedTimeMs()), + NiceBytes(CloudValue.Size()), + Key); + + Value.Value = CloudValue; + } + else + { + spdlog::debug("upstream miss after {:5}! {}", zen::NiceTimeSpanMs(Timer.getElapsedTimeMs()), Key); + } + } + + if (Success && (Value.Value.Size() <= m_InMemoryBlobSizeThreshold)) + { + m_cache.Put(Key, Value); + } + } + + if (!Success) + { + Request.WriteResponse(zen::HttpResponse::NotFound); + } + else + { + if (Verb == zen::HttpVerb::kHead) + { + Request.SetSuppressResponseBody(); + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Value.Value); + } + else + { + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Value.Value); + } + } + } + break; + + case kPut: + { + if (zen::IoBuffer Body = Request.ReadPayload()) + { + CacheValue Value; + Value.Value = Body; + + if (Value.Value.Size() <= m_InMemoryBlobSizeThreshold) + { + m_cache.Put(Key, Value); + } + + // m_cache_.Put(Key, Value); + + CloudCacheSession Session(m_Cloud); + + zen::Stopwatch Timer; + + Session.Put("default", Key, Value.Value); + + spdlog::debug("upstream PUT took {:5} {:6}! {}", + zen::NiceTimeSpanMs(Timer.getElapsedTimeMs()), + NiceBytes(Value.Value.Size()), + Key); + + Request.WriteResponse(zen::HttpResponse::Created); + } + else + { + return; + } + } + break; + + case kDelete: + // should this do anything? + return Request.WriteResponse(zen::HttpResponse::OK); + + case kPost: + break; + + default: + break; + } +} + +} // namespace zen diff --git a/zenserver/cache/kvcache.h b/zenserver/cache/kvcache.h new file mode 100644 index 000000000..e601582a4 --- /dev/null +++ b/zenserver/cache/kvcache.h @@ -0,0 +1,38 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/httpserver.h> + +#include "cachestore.h" +#include "upstream/jupiter.h" + +namespace zen { + +/** + * Generic HTTP K/V cache - can be consumed via legacy DDC interfaces, with + * no key format conventions. Values are blobs + */ + +class HttpKvCacheService : public zen::HttpService +{ +public: + HttpKvCacheService(); + ~HttpKvCacheService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + MemoryCacheStore m_cache; + FileCacheStore m_cache_{"E:\\Local-DDC-Write", "E:\\Local-DDC" /* Read */}; + RefPtr<CloudCacheClient> m_Cloud; + uint64_t m_InMemoryBlobSizeThreshold = 16384; + uint64_t m_FileBlobSizeThreshold = 16 * 1024 * 1024; + + struct AccessTracker; + + std::unique_ptr<AccessTracker> m_AccessTracker; +}; + +} // namespace zen diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp new file mode 100644 index 000000000..0d62f297c --- /dev/null +++ b/zenserver/cache/structuredcache.cpp @@ -0,0 +1,129 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/fmtutils.h> +#include <zencore/httpserver.h> + +#include "cachestore.h" +#include "structuredcache.h" +#include "upstream/jupiter.h" + +#include <spdlog/spdlog.h> +#include <filesystem> + +namespace zen { + +HttpStructuredCacheService::HttpStructuredCacheService(std::filesystem::path RootPath, zen::CasStore& InStore) +: m_CasStore(InStore) +, m_CacheStore(InStore, RootPath) +{ + spdlog::info("initializing structured cache at '{}'", RootPath); +} + +HttpStructuredCacheService::~HttpStructuredCacheService() +{ + spdlog::info("closing structured cache"); +} + +const char* +HttpStructuredCacheService::BaseUri() const +{ + return "/z$/"; +} + +void +HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) +{ + CacheRef Ref; + + if (!ValidateUri(Request, /* out */ Ref)) + { + return Request.WriteResponse(zen::HttpResponse::BadRequest); // invalid URL + } + + switch (auto Verb = Request.RequestVerb()) + { + using enum zen::HttpVerb; + + case kHead: + case kGet: + { + CacheValue Value; + bool Success = m_CacheStore.Get(Ref.BucketSegment, Ref.HashKey, /* out */ Value); + + if (!Success) + { + Request.WriteResponse(zen::HttpResponse::NotFound); + } + else + { + if (Verb == kHead) + { + Request.SetSuppressResponseBody(); + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Value.Value); + } + else + { + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Value.Value); + } + } + } + break; + + case kPut: + { + if (zen::IoBuffer Body = Request.ReadPayload()) + { + CacheValue Value; + Value.Value = Body; + + m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, Value); + + Request.WriteResponse(zen::HttpResponse::Created); + } + else + { + return; + } + } + break; + + case kPost: + break; + + default: + break; + } +} + +[[nodiscard]] bool +HttpStructuredCacheService::ValidateUri(zen::HttpServerRequest& Request, CacheRef& OutRef) +{ + std::string_view Key = Request.RelativeUri(); + std::string_view::size_type BucketSplitOffset = Key.find_last_of('/'); + + if (BucketSplitOffset == std::string_view::npos) + { + return false; + } + + OutRef.BucketSegment = Key.substr(0, BucketSplitOffset); + std::string_view HashSegment = Key.substr(BucketSplitOffset + 1); + + if (HashSegment.size() != (2 * sizeof OutRef.HashKey.Hash)) + { + return false; + } + + bool IsOk = zen::ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); + + if (!IsOk) + { + return false; + } + + return true; +} + +} // namespace zen diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h new file mode 100644 index 000000000..d0646e6e9 --- /dev/null +++ b/zenserver/cache/structuredcache.h @@ -0,0 +1,40 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/httpserver.h> + +#include "cachestore.h" +#include "upstream/jupiter.h" + +namespace zen { + +/** + * New-style cache service. Imposes constraints on keys, supports blobs and + * structured values + */ + +class HttpStructuredCacheService : public zen::HttpService +{ +public: + HttpStructuredCacheService(std::filesystem::path RootPath, zen::CasStore& InStore); + ~HttpStructuredCacheService(); + + virtual const char* BaseUri() const override; + + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + struct CacheRef + { + std::string BucketSegment; + IoHash HashKey; + }; + + [[nodiscard]] bool ValidateUri(zen::HttpServerRequest& Request, CacheRef& OutRef); + + zen::CasStore& m_CasStore; + ZenCacheStore m_CacheStore; +}; + +} // namespace zen diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp new file mode 100644 index 000000000..4afcf21a6 --- /dev/null +++ b/zenserver/casstore.cpp @@ -0,0 +1,155 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "casstore.h" + +#include <zencore/streamutil.h> + +#include <spdlog/spdlog.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) +{ + m_Router.AddPattern("cas", "([0-9A-Fa-f]{40})"); + + m_Router.RegisterRoute( + "batch", + [this](HttpRouterRequest& Req) { + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + IoBuffer Payload = ServerRequest.ReadPayload(); + uint64_t EntryCount = Payload.Size() / sizeof(IoHash); + + if ((EntryCount * sizeof(IoHash)) != Payload.Size()) + { + return ServerRequest.WriteResponse(HttpResponse::BadRequest); + } + + const IoHash* Hashes = reinterpret_cast<const IoHash*>(Payload.Data()); + std::vector<IoBuffer> Values; + + MemoryOutStream HeaderStream; + BinaryWriter HeaderWriter(HeaderStream); + + Values.emplace_back(); // Placeholder for header + + // Build response header + HeaderWriter << uint32_t(0x12340000) << uint32_t(0); + + for (uint64_t i = 0; i < EntryCount; ++i) + { + IoHash ChunkHash = Hashes[i]; + IoBuffer Value = m_CasStore.FindChunk(ChunkHash); + + if (Value) + { + Values.emplace_back(std::move(Value)); + HeaderWriter << ChunkHash << uint64_t(Value.Size()); + } + } + + // Make real header + + const_cast<uint32_t*>(reinterpret_cast<const uint32_t*>(HeaderStream.Data()))[1] = uint32_t(Values.size() - 1); + + Values[0] = IoBufferBuilder::MakeCloneFromMemory(HeaderStream.Data(), HeaderStream.Size()); + + ServerRequest.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Values); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{cas}", + [this](HttpRouterRequest& Req) { + IoHash Hash = IoHash::FromHexString(Req.GetCapture(1)); + spdlog::debug("CAS request for {}", Hash); + + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kGet: + case HttpVerb::kHead: + { + if (IoBuffer Value = m_CasStore.FindChunk(Hash)) + { + return ServerRequest.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Value); + } + + return ServerRequest.WriteResponse(HttpResponse::NotFound); + } + break; + + case HttpVerb::kPut: + { + IoBuffer Payload = ServerRequest.ReadPayload(); + IoHash PayloadHash = IoHash::HashMemory(Payload.Data(), Payload.Size()); + + // URI hash must match content hash + if (PayloadHash != Hash) + { + return ServerRequest.WriteResponse(HttpResponse::BadRequest); + } + + m_CasStore.InsertChunk(Payload.Data(), Payload.Size(), PayloadHash); + + return ServerRequest.WriteResponse(HttpResponse::OK); + } + break; + } + }, + HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kHead); +} + +const char* +HttpCasService::BaseUri() const +{ + return "/cas/"; +} + +void +HttpCasService::HandleRequest(zen::HttpServerRequest& Request) +{ + if (Request.RelativeUri().empty()) + { + // Root URI request + + switch (Request.RequestVerb()) + { + case HttpVerb::kPut: + case HttpVerb::kPost: + { + IoBuffer Payload = Request.ReadPayload(); + IoHash PayloadHash = IoHash::HashMemory(Payload.Data(), Payload.Size()); + + spdlog::debug("CAS POST request for {} ({} bytes)", PayloadHash, Payload.Size()); + + auto InsertResult = m_CasStore.InsertChunk(Payload.Data(), Payload.Size(), PayloadHash); + + if (InsertResult.New) + { + return Request.WriteResponse(HttpResponse::Created); + } + else + { + return Request.WriteResponse(HttpResponse::OK); + } + } + break; + + case HttpVerb::kGet: + case HttpVerb::kHead: + break; + + default: + break; + } + } + else + { + m_Router.HandleRequest(Request); + } +} + +} // namespace zen diff --git a/zenserver/casstore.h b/zenserver/casstore.h new file mode 100644 index 000000000..7166f796e --- /dev/null +++ b/zenserver/casstore.h @@ -0,0 +1,34 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/httpserver.h> +#include <zenstore/cas.h> + +namespace zen { + +/** + * Simple CAS store HTTP endpoint + * + * Note that since this does not end up pinning any of the chunks it's only really useful for a small subset of use cases where you know a + * chunk exists in the underlying CAS store. Thus it's mainly useful for internal use when communicating between Zen store instances + * + * Using this interface for adding CAS chunks makes little sense except for testing purposes as garbage collection may reap anything you add + * before anything ever gets to access it + */ + +class HttpCasService : public HttpService +{ +public: + explicit HttpCasService(CasStore& Store); + ~HttpCasService() = default; + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + CasStore& m_CasStore; + HttpRequestRouter m_Router; +}; + +} // namespace zen diff --git a/zenserver/config.cpp b/zenserver/config.cpp new file mode 100644 index 000000000..027427528 --- /dev/null +++ b/zenserver/config.cpp @@ -0,0 +1,157 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "config.h" + +#include "diag/logging.h" + +#include <zencore/fmtutils.h> +#include <zencore/iobuffer.h> +#include <zencore/string.h> + +#pragma warning(push) +#pragma warning(disable : 4267) // warning C4267: '=': conversion from 'size_t' to 'US', possible loss of data +#include <cxxopts.hpp> +#pragma warning(pop) + +#include <fmt/format.h> +#include <spdlog/spdlog.h> +#include <sol/sol.hpp> + +#if ZEN_PLATFORM_WINDOWS + +// Used for getting My Documents for default data directory +# include <ShlObj.h> +# pragma comment(lib, "shell32.lib") + +std::filesystem::path +PickDefaultStateDirectory() +{ + // Pick sensible default + + WCHAR myDocumentsDir[MAX_PATH]; + HRESULT hRes = SHGetFolderPathW(NULL, + CSIDL_PERSONAL /* My Documents */, + NULL, + SHGFP_TYPE_CURRENT, + /* out */ myDocumentsDir); + + if (SUCCEEDED(hRes)) + { + wcscat_s(myDocumentsDir, L"\\zen"); + + return myDocumentsDir; + } + + return L""; +} + +#else + +std::filesystem::path +PickDefaultStateDirectory() +{ + return std::filesystem::path("~/.zen"); +} + +#endif + +void +ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions) +{ + cxxopts::Options options("zenserver", "Zen Server"); + options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug)->default_value("false")); + options.add_options()("help", "Show command line help"); + options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(GlobalOptions.IsTest)->default_value("false")); + options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(GlobalOptions.LogId)); + options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::filesystem::path>(GlobalOptions.DataDir)); + + options + .add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value<int>(GlobalOptions.OwnerPid), "<identifier>"); + options.add_option("lifetime", + "", + "child-id", + "Specify id which can be used to signal parent", + cxxopts::value<std::string>(GlobalOptions.ChildId), + "<identifier>"); + + options.add_option("network", + "p", + "port", + "Select HTTP port", + cxxopts::value<int>(GlobalOptions.BasePort)->default_value("1337"), + "<port number>"); + + try + { + auto result = options.parse(argc, argv); + + if (result.count("help")) + { + ConsoleLog().info("{}", options.help()); + + exit(0); + } + } + catch (cxxopts::OptionParseException& e) + { + ConsoleLog().error("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help()); + + throw; + } + + if (GlobalOptions.DataDir.empty()) + { + GlobalOptions.DataDir = PickDefaultStateDirectory(); + } +} + +void +ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& ServiceConfig) +{ + using namespace fmt::literals; + + std::filesystem::path ConfigScript = DataRoot / "zen_cfg.lua"; + zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(ConfigScript.native().c_str()); + + if (LuaScript) + { + sol::state lua; + + // Provide some context to help derive defaults + lua.set("dataroot", DataRoot.native()); + + lua.open_libraries(sol::lib::base); + + // We probably want to limit the scope of this so the script won't see + // any more than it needs to + + lua.set_function("getenv", [&](const std::string env) -> sol::object { + std::wstring EnvVarValue; + size_t RequiredSize = 0; + std::wstring EnvWide = zen::Utf8ToWide(env); + _wgetenv_s(&RequiredSize, nullptr, 0, EnvWide.c_str()); + + if (RequiredSize == 0) + return sol::make_object(lua, sol::lua_nil); + + EnvVarValue.resize(RequiredSize); + _wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str()); + return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str())); + }); + + try + { + sol::load_result config = lua.load(std::string_view((const char*)LuaScript.Data(), LuaScript.Size()), "zencfg"); + config(); + } + catch (std::exception& e) + { + spdlog::error("config script failure: {}", e.what()); + + throw std::exception("fatal zen global config script ({}) failure: {}"_format(ConfigScript, e.what()).c_str()); + } + ServiceConfig.LegacyCacheEnabled = lua["legacycache"]["enable"]; + const std::string path = lua["legacycache"]["readpath"]; + ServiceConfig.StructuredCacheEnabled = lua["structuredcache"]["enable"]; + } +} diff --git a/zenserver/config.h b/zenserver/config.h new file mode 100644 index 000000000..c96dc139a --- /dev/null +++ b/zenserver/config.h @@ -0,0 +1,28 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <filesystem> +#include <string> + +struct ZenServerOptions +{ + bool IsDebug = false; + bool IsTest = false; + int BasePort = 1337; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::string LogId; // Id for tagging log output + std::filesystem::path DataDir; // Root directory for state (used for testing) + std::string FlockId; // Id for grouping test instances into sets +}; + +void ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions); + +struct ZenServiceConfig +{ + bool LegacyCacheEnabled = false; + bool StructuredCacheEnabled = true; +}; + +void ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& ServiceConfig); diff --git a/zenserver/diag/crashreport.cpp b/zenserver/diag/crashreport.cpp new file mode 100644 index 000000000..03e74ca5c --- /dev/null +++ b/zenserver/diag/crashreport.cpp @@ -0,0 +1,85 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "crashreport.h" + +#include <zencore/filesystem.h> +#include <zencore/zencore.h> + +#include <client/windows/handler/exception_handler.h> + +#include <filesystem> + +// A callback function to run after the minidump has been written. +// minidump_id is a unique id for the dump, so the minidump +// file is <dump_path>\<minidump_id>.dmp. context is the parameter supplied +// by the user as callback_context when the handler was created. exinfo +// points to the exception record, or NULL if no exception occurred. +// succeeded indicates whether a minidump file was successfully written. +// assertion points to information about an assertion if the handler was +// invoked by an assertion. +// +// If an exception occurred and the callback returns true, Breakpad will treat +// the exception as fully-handled, suppressing any other handlers from being +// notified of the exception. If the callback returns false, Breakpad will +// treat the exception as unhandled, and allow another handler to handle it. +// If there are no other handlers, Breakpad will report the exception to the +// system as unhandled, allowing a debugger or native crash dialog the +// opportunity to handle the exception. Most callback implementations +// should normally return the value of |succeeded|, or when they wish to +// not report an exception of handled, false. Callbacks will rarely want to +// return true directly (unless |succeeded| is true). +// +// For out-of-process dump generation, dump path and minidump ID will always +// be NULL. In case of out-of-process dump generation, the dump path and +// minidump id are controlled by the server process and are not communicated +// back to the crashing process. + +static bool +CrashMinidumpCallback(const wchar_t* dump_path, + const wchar_t* minidump_id, + void* context, + EXCEPTION_POINTERS* exinfo, + MDRawAssertionInfo* assertion, + bool succeeded) +{ + ZEN_UNUSED(dump_path, minidump_id, context, exinfo, assertion, succeeded); + + // TODO! + return succeeded; +} + +// A callback function to run before Breakpad performs any substantial +// processing of an exception. A FilterCallback is called before writing +// a minidump. context is the parameter supplied by the user as +// callback_context when the handler was created. exinfo points to the +// exception record, if any; assertion points to assertion information, +// if any. +// +// If a FilterCallback returns true, Breakpad will continue processing, +// attempting to write a minidump. If a FilterCallback returns false, +// Breakpad will immediately report the exception as unhandled without +// writing a minidump, allowing another handler the opportunity to handle it. + +bool +CrashFilterCallback(void* context, EXCEPTION_POINTERS* exinfo, MDRawAssertionInfo* assertion) +{ + ZEN_UNUSED(context, exinfo, assertion); + + // Yes, write a dump + return false; +} + +void +InitializeCrashReporting(const std::filesystem::path& DumpPath) +{ + // handler_types specifies the types of handlers that should be installed. + + zen::CreateDirectories(DumpPath); + + static google_breakpad::ExceptionHandler _(DumpPath.native().c_str(), // Dump path + CrashFilterCallback, // Filter Callback + CrashMinidumpCallback, // Minidump callback + nullptr, // Callback context + google_breakpad::ExceptionHandler::HANDLER_ALL // Handler Types + ); +} diff --git a/zenserver/diag/crashreport.h b/zenserver/diag/crashreport.h new file mode 100644 index 000000000..6369d1cf5 --- /dev/null +++ b/zenserver/diag/crashreport.h @@ -0,0 +1,9 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +namespace std::filesystem { +class path; +} + +void InitializeCrashReporting(const std::filesystem::path& DumpPath); diff --git a/zenserver/diag/diagsvcs.h b/zenserver/diag/diagsvcs.h new file mode 100644 index 000000000..84f8d22ee --- /dev/null +++ b/zenserver/diag/diagsvcs.h @@ -0,0 +1,103 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/httpserver.h> +#include <zencore/iobuffer.h> + +////////////////////////////////////////////////////////////////////////// + +class HttpTestService : public zen::HttpService +{ + uint32_t LogPoint = 0; + +public: + HttpTestService() {} + ~HttpTestService() = default; + + virtual const char* BaseUri() const override { return "/test/"; } + + virtual void HandleRequest(zen::HttpServerRequest& Request) override + { + using namespace std::literals; + + auto Uri = Request.RelativeUri(); + + if (Uri == "hello"sv) + { + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kText, u8"hello world!"sv); + + // OutputLogMessageInternal(&LogPoint, 0, 0); + } + else if (Uri == "1K"sv) + { + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, m_1k); + } + else if (Uri == "1M"sv) + { + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, m_1m); + } + else if (Uri == "1M_1k"sv) + { + std::vector<zen::IoBuffer> Buffers; + Buffers.reserve(1024); + + for (int i = 0; i < 1024; ++i) + { + Buffers.push_back(m_1k); + } + + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers); + } + else if (Uri == "1G"sv) + { + std::vector<zen::IoBuffer> Buffers; + Buffers.reserve(1024); + + for (int i = 0; i < 1024; ++i) + { + Buffers.push_back(m_1m); + } + + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers); + } + else if (Uri == "1G_1k"sv) + { + std::vector<zen::IoBuffer> Buffers; + Buffers.reserve(1024 * 1024); + + for (int i = 0; i < 1024 * 1024; ++i) + { + Buffers.push_back(m_1k); + } + + Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Buffers); + } + } + +private: + zen::IoBuffer m_1m{1024 * 1024}; + zen::IoBuffer m_1k{m_1m, 0u, 1024}; +}; + +class HttpHealthService : public zen::HttpService +{ +public: + HttpHealthService() = default; + ~HttpHealthService() = default; + + virtual const char* BaseUri() const override { return "/health/"; } + + virtual void HandleRequest(zen::HttpServerRequest& Request) override + { + using namespace std::literals; + + switch (Request.RequestVerb()) + { + case zen::HttpVerb::kGet: + return Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kText, u8"OK!"sv); + } + } + +private: +}; diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp new file mode 100644 index 000000000..2bf0e50aa --- /dev/null +++ b/zenserver/diag/logging.cpp @@ -0,0 +1,204 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "logging.h" + +#include "config.h" + +#include <spdlog/pattern_formatter.h> +#include <spdlog/sinks/ansicolor_sink.h> +#include <spdlog/sinks/stdout_color_sinks.h> +#include <spdlog/spdlog.h> +#include <memory> + +// Custom logging -- test code, this should be tweaked + +namespace logging { + +using namespace spdlog; +using namespace spdlog::details; +using namespace std::literals; + +class full_formatter final : public spdlog::formatter +{ +public: + full_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch) : m_Epoch(Epoch), m_LogId(LogId) {} + + virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_formatter>(m_LogId, m_Epoch); } + + static constexpr bool UseDate = false; + + virtual void format(const details::log_msg& msg, memory_buf_t& dest) override + { + using std::chrono::duration_cast; + using std::chrono::milliseconds; + using std::chrono::seconds; + + if constexpr (UseDate) + { + auto secs = std::chrono::duration_cast<seconds>(msg.time.time_since_epoch()); + if (secs != m_LastLogSecs) + { + m_CachedTm = os::localtime(log_clock::to_time_t(msg.time)); + m_LastLogSecs = secs; + } + } + + const auto& tm_time = m_CachedTm; + + // cache the date/time part for the next second. + auto duration = msg.time - m_Epoch; + auto secs = duration_cast<seconds>(duration); + + if (m_CacheTimestamp != secs || m_CachedDatetime.size() == 0) + { + m_CachedDatetime.clear(); + m_CachedDatetime.push_back('['); + + if constexpr (UseDate) + { + fmt_helper::append_int(tm_time.tm_year + 1900, m_CachedDatetime); + m_CachedDatetime.push_back('-'); + + fmt_helper::pad2(tm_time.tm_mon + 1, m_CachedDatetime); + m_CachedDatetime.push_back('-'); + + fmt_helper::pad2(tm_time.tm_mday, m_CachedDatetime); + m_CachedDatetime.push_back(' '); + + fmt_helper::pad2(tm_time.tm_hour, m_CachedDatetime); + m_CachedDatetime.push_back(':'); + + fmt_helper::pad2(tm_time.tm_min, m_CachedDatetime); + m_CachedDatetime.push_back(':'); + + fmt_helper::pad2(tm_time.tm_sec, m_CachedDatetime); + } + else + { + int Count = int(secs.count()); + + const int LogSecs = Count % 60; + Count /= 60; + + const int LogMins = Count % 60; + Count /= 60; + + const int LogHours = Count; + + fmt_helper::pad2(LogHours, m_CachedDatetime); + m_CachedDatetime.push_back(':'); + fmt_helper::pad2(LogMins, m_CachedDatetime); + m_CachedDatetime.push_back(':'); + fmt_helper::pad2(LogSecs, m_CachedDatetime); + } + + m_CachedDatetime.push_back('.'); + + m_CacheTimestamp = secs; + } + + dest.append(m_CachedDatetime.begin(), m_CachedDatetime.end()); + + auto millis = fmt_helper::time_fraction<milliseconds>(msg.time); + fmt_helper::pad3(static_cast<uint32_t>(millis.count()), dest); + dest.push_back(']'); + dest.push_back(' '); + + if (!m_LogId.empty()) + { + dest.push_back('['); + fmt_helper::append_string_view(m_LogId, dest); + dest.push_back(']'); + dest.push_back(' '); + } + + // append logger name if exists + if (msg.logger_name.size() > 0) + { + dest.push_back('['); + fmt_helper::append_string_view(msg.logger_name, dest); + dest.push_back(']'); + dest.push_back(' '); + } + + dest.push_back('['); + // wrap the level name with color + msg.color_range_start = dest.size(); + fmt_helper::append_string_view(level::to_string_view(msg.level), dest); + msg.color_range_end = dest.size(); + dest.push_back(']'); + dest.push_back(' '); + + // add source location if present + if (!msg.source.empty()) + { + dest.push_back('['); + const char* filename = details::short_filename_formatter<details::null_scoped_padder>::basename(msg.source.filename); + fmt_helper::append_string_view(filename, dest); + dest.push_back(':'); + fmt_helper::append_int(msg.source.line, dest); + dest.push_back(']'); + dest.push_back(' '); + } + + fmt_helper::append_string_view(msg.payload, dest); + fmt_helper::append_string_view("\n"sv, dest); + } + +private: + std::chrono::time_point<std::chrono::system_clock> m_Epoch; + std::tm m_CachedTm; + std::chrono::seconds m_LastLogSecs; + std::chrono::seconds m_CacheTimestamp{0}; + memory_buf_t m_CachedDatetime; + std::string m_LogId; +}; + +} // namespace logging + +bool +EnableVTMode() +{ + // Set output mode to handle virtual terminal sequences + HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); + if (hOut == INVALID_HANDLE_VALUE) + { + return false; + } + + DWORD dwMode = 0; + if (!GetConsoleMode(hOut, &dwMode)) + { + return false; + } + + dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + if (!SetConsoleMode(hOut, dwMode)) + { + return false; + } + + return true; +} + +void +InitializeLogging(const ZenServerOptions& GlobalOptions) +{ + EnableVTMode(); + + auto& sinks = spdlog::default_logger()->sinks(); + sinks.clear(); + sinks.push_back(std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>()); + spdlog::set_level(spdlog::level::debug); + spdlog::set_formatter(std::make_unique<logging::full_formatter>(GlobalOptions.LogId, std::chrono::system_clock::now())); +} + +spdlog::logger& +ConsoleLog() +{ + static auto ConLogger = spdlog::stdout_color_mt("console"); + + ConLogger->set_pattern("%v"); + + return *ConLogger; +} diff --git a/zenserver/diag/logging.h b/zenserver/diag/logging.h new file mode 100644 index 000000000..1b1813913 --- /dev/null +++ b/zenserver/diag/logging.h @@ -0,0 +1,11 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <spdlog/spdlog.h> + +struct ZenServerOptions; + +void InitializeLogging(const ZenServerOptions& GlobalOptions); + +spdlog::logger& ConsoleLog(); diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp new file mode 100644 index 000000000..f44e50945 --- /dev/null +++ b/zenserver/experimental/usnjournal.cpp @@ -0,0 +1,341 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "usnjournal.h" + +#include <zencore/except.h> +#include <zencore/timer.h> +#include <zencore/zencore.h> + +#include <spdlog/spdlog.h> + +#include <atlfile.h> + +#include <filesystem> + +namespace zen { + +UsnJournalReader::UsnJournalReader() +{ +} + +UsnJournalReader::~UsnJournalReader() +{ + delete[] m_JournalReadBuffer; +} + +bool +UsnJournalReader::Initialize(std::filesystem::path VolumePath) +{ + TCHAR VolumeName[MAX_PATH]; + TCHAR VolumePathName[MAX_PATH]; + + { + auto NativePath = VolumePath.native(); + BOOL Success = GetVolumePathName(NativePath.c_str(), VolumePathName, ZEN_ARRAY_COUNT(VolumePathName)); + + if (!Success) + { + zen::ThrowSystemException("GetVolumePathName failed"); + } + + Success = GetVolumeNameForVolumeMountPoint(VolumePathName, VolumeName, ZEN_ARRAY_COUNT(VolumeName)); + + if (!Success) + { + zen::ThrowSystemException("GetVolumeNameForVolumeMountPoint failed"); + } + + // Chop off trailing slash since we want to open a volume handle, not a handle to the volume root directory + + const size_t VolumeNameLength = wcslen(VolumeName); + + if (VolumeNameLength) + { + VolumeName[VolumeNameLength - 1] = '\0'; + } + } + + m_VolumeHandle = CreateFile(VolumeName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, /* no custom security */ + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + nullptr); /* template */ + + if (m_VolumeHandle == INVALID_HANDLE_VALUE) + { + ThrowSystemException("Volume handle open failed"); + } + + // Figure out which file system is in use for volume + + { + WCHAR InfoVolumeName[MAX_PATH + 1]{}; + WCHAR FileSystemName[MAX_PATH + 1]{}; + DWORD MaximumComponentLength = 0; + DWORD FileSystemFlags = 0; + + BOOL Success = GetVolumeInformationByHandleW(m_VolumeHandle, + InfoVolumeName, + MAX_PATH + 1, + NULL, + &MaximumComponentLength, + &FileSystemFlags, + FileSystemName, + ZEN_ARRAY_COUNT(FileSystemName)); + + if (!Success) + { + ThrowSystemException("Failed to get volume information"); + } + + spdlog::debug("File system type is {}", WideToUtf8(FileSystemName)); + + if (wcscmp(L"ReFS", FileSystemName) == 0) + { + m_FileSystemType = FileSystemType::ReFS; + } + else if (wcscmp(L"NTFS", FileSystemName) == 0) + { + m_FileSystemType = FileSystemType::NTFS; + } + else + { + // Unknown file system type! + } + } + + // Determine if volume is on fast storage, where seeks aren't so expensive + + { + STORAGE_PROPERTY_QUERY StorageQuery{}; + StorageQuery.PropertyId = StorageDeviceSeekPenaltyProperty; + StorageQuery.QueryType = PropertyStandardQuery; + DWORD BytesWritten; + DEVICE_SEEK_PENALTY_DESCRIPTOR Result{}; + + if (DeviceIoControl(m_VolumeHandle, + IOCTL_STORAGE_QUERY_PROPERTY, + &StorageQuery, + sizeof(StorageQuery), + &Result, + sizeof(Result), + &BytesWritten, + nullptr)) + { + m_IncursSeekPenalty = !!Result.IncursSeekPenalty; + } + } + + // Query Journal + + USN_JOURNAL_DATA_V2 UsnData{}; + + { + DWORD BytesWritten = 0; + + const BOOL Success = + DeviceIoControl(m_VolumeHandle, FSCTL_QUERY_USN_JOURNAL, nullptr, 0, &UsnData, sizeof UsnData, &BytesWritten, nullptr); + + if (!Success) + { + switch (DWORD Error = GetLastError()) + { + case ERROR_JOURNAL_NOT_ACTIVE: + spdlog::info("No USN journal active on drive"); + + // TODO: optionally activate USN journal on drive? + + ThrowSystemException(HRESULT_FROM_WIN32(Error), "No USN journal active on drive"); + break; + + default: + ThrowSystemException(HRESULT_FROM_WIN32(Error), "FSCTL_QUERY_USN_JOURNAL failed"); + } + } + } + + m_JournalReadBuffer = new uint8_t[m_ReadBufferSize]; + + // Catch up to USN start + + CAtlFile VolumeRootDir; + HRESULT hRes = + VolumeRootDir.Create(VolumePathName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS); + + ThrowIfFailed(hRes, "Failed to open handle to volume root"); + + FILE_ID_INFO FileInformation{}; + BOOL Success = GetFileInformationByHandleEx(VolumeRootDir, FileIdInfo, &FileInformation, sizeof FileInformation); + + if (!Success) + { + ThrowSystemException("GetFileInformationByHandleEx failed"); + } + + const Frn VolumeRootFrn = FileInformation.FileId; + + // Enumerate MFT (but not for ReFS) + + if (m_FileSystemType == FileSystemType::NTFS) + { + spdlog::info("Enumerating MFT for {}", WideToUtf8(VolumePathName)); + + zen::Stopwatch Timer; + uint64_t MftBytesProcessed = 0; + + MFT_ENUM_DATA_V1 MftEnumData{.StartFileReferenceNumber = 0, .LowUsn = 0, .HighUsn = 0, .MinMajorVersion = 2, .MaxMajorVersion = 3}; + + BYTE MftBuffer[64 * 1024 + sizeof(DWORDLONG)]; + DWORD BytesWritten = 0; + + for (;;) + { + Success = DeviceIoControl(m_VolumeHandle, + FSCTL_ENUM_USN_DATA, + &MftEnumData, + sizeof MftEnumData, + MftBuffer, + sizeof MftBuffer, + &BytesWritten, + nullptr); + + if (!Success) + { + DWORD Error = GetLastError(); + + if (Error == ERROR_HANDLE_EOF) + { + break; + } + + ThrowSystemException(HRESULT_FROM_WIN32(Error), "FSCTL_ENUM_USN_DATA failed"); + } + + void* BufferEnd = (void*)&MftBuffer[BytesWritten]; + + // The enumeration call returns the next FRN ahead of the other data in the buffer + MftEnumData.StartFileReferenceNumber = ((DWORDLONG*)MftBuffer)[0]; + + PUSN_RECORD_UNION CommonRecord = PUSN_RECORD_UNION(&((DWORDLONG*)MftBuffer)[1]); + + while (CommonRecord < BufferEnd) + { + switch (CommonRecord->Header.MajorVersion) + { + case 2: + { + USN_RECORD_V2& Record = CommonRecord->V2; + + const Frn FileReference = Record.FileReferenceNumber; + const Frn ParentReference = Record.ParentFileReferenceNumber; + std::wstring_view FileName{Record.FileName, Record.FileNameLength}; + } + break; + case 3: + { + USN_RECORD_V3& Record = CommonRecord->V3; + + const Frn FileReference = Record.FileReferenceNumber; + const Frn ParentReference = Record.ParentFileReferenceNumber; + std::wstring_view FileName{Record.FileName, Record.FileNameLength}; + } + break; + case 4: + { + // This captures file modification ranges. We do not yet support this however + USN_RECORD_V4& Record = CommonRecord->V4; + } + break; + } + + const DWORD RecordLength = CommonRecord->Header.RecordLength; + CommonRecord = PUSN_RECORD_UNION(((uint8_t*)CommonRecord) + RecordLength); + MftBytesProcessed += RecordLength; + } + } + + const auto ElapsedMs = Timer.getElapsedTimeMs(); + + spdlog::info("MFT enumeration of {} completed after {} ({})", + zen::NiceBytes(MftBytesProcessed), + zen::NiceTimeSpanMs(ElapsedMs), + zen::NiceByteRate(MftBytesProcessed, ElapsedMs)); + } + + // Populate by traversal + if (m_FileSystemType == FileSystemType::ReFS) + { + uint64_t FileInfoBuffer[8 * 1024]; + + FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; + bool Continue = true; + + while (Continue) + { + Success = GetFileInformationByHandleEx(VolumeRootDir, FibClass, FileInfoBuffer, sizeof FileInfoBuffer); + FibClass = FileIdBothDirectoryInfo; // Set up for next iteration + + uint64_t EntryOffset = 0; + + if (!Success) + { + // Report failure? + + break; + } + + do + { + const FILE_ID_BOTH_DIR_INFO* DirInfo = + reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset); + + const uint64_t NextOffset = DirInfo->NextEntryOffset; + + if (NextOffset == 0) + { + if (EntryOffset == 0) + { + // First and last - end of iteration + Continue = false; + } + break; + } + + if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + // TODO Directory + } + else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE) + { + // TODO Device + } + else + { + // TODO File + } + + EntryOffset += DirInfo->NextEntryOffset; + } while (EntryOffset); + } + } + + // Initialize journal reading + + m_ReadUsnJournalData = {.StartUsn = UsnData.FirstUsn, + .ReasonMask = USN_REASON_BASIC_INFO_CHANGE | USN_REASON_CLOSE | USN_REASON_DATA_EXTEND | + USN_REASON_DATA_OVERWRITE | USN_REASON_DATA_TRUNCATION | USN_REASON_FILE_CREATE | + USN_REASON_FILE_DELETE | USN_REASON_HARD_LINK_CHANGE | USN_REASON_RENAME_NEW_NAME | + USN_REASON_RENAME_OLD_NAME | USN_REASON_REPARSE_POINT_CHANGE, + .ReturnOnlyOnClose = true, + .Timeout = 0, + .BytesToWaitFor = 0, + .UsnJournalID = UsnData.UsnJournalID, + .MinMajorVersion = 0, + .MaxMajorVersion = 0}; + + return false; +} + +} // namespace zen diff --git a/zenserver/experimental/usnjournal.h b/zenserver/experimental/usnjournal.h new file mode 100644 index 000000000..9c1008d52 --- /dev/null +++ b/zenserver/experimental/usnjournal.h @@ -0,0 +1,62 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/windows.h> + +#include <winioctl.h> + +#include <filesystem> + +namespace zen { + +class UsnJournalReader +{ +public: + UsnJournalReader(); + ~UsnJournalReader(); + + bool Initialize(std::filesystem::path VolumePath); + +private: + void* m_VolumeHandle; + READ_USN_JOURNAL_DATA_V1 m_ReadUsnJournalData; + bool m_IncursSeekPenalty = true; + + uint8_t* m_JournalReadBuffer = nullptr; + uint64_t m_ReadBufferSize = 64 * 1024; + + struct Frn + { + uint8_t IdBytes[16]; + + Frn() = default; + + Frn(const FILE_ID_128& Rhs) { memcpy(IdBytes, Rhs.Identifier, sizeof IdBytes); } + Frn& operator=(const FILE_ID_128& Rhs) { memcpy(IdBytes, Rhs.Identifier, sizeof IdBytes); } + + Frn(const uint64_t& Rhs) + { + memcpy(IdBytes, &Rhs, sizeof Rhs); + memset(&IdBytes[8], 0, 8); + } + + Frn& operator=(const uint64_t& Rhs) + { + memcpy(IdBytes, &Rhs, sizeof Rhs); + memset(&IdBytes[8], 0, 8); + } + + std::strong_ordering operator<=>(const Frn&) const = default; + }; + + enum class FileSystemType + { + ReFS, + NTFS + }; + + FileSystemType m_FileSystemType = FileSystemType::NTFS; +}; + +} // namespace zen diff --git a/zenserver/experimental/vfs.cpp b/zenserver/experimental/vfs.cpp new file mode 100644 index 000000000..1af9d70a7 --- /dev/null +++ b/zenserver/experimental/vfs.cpp @@ -0,0 +1,3 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "vfs.h" diff --git a/zenserver/experimental/vfs.h b/zenserver/experimental/vfs.h new file mode 100644 index 000000000..1aeefe481 --- /dev/null +++ b/zenserver/experimental/vfs.h @@ -0,0 +1,5 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp new file mode 100644 index 000000000..0dc0da1ae --- /dev/null +++ b/zenserver/projectstore.cpp @@ -0,0 +1,1547 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "projectstore.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/stream.h> +#include <zencore/string.h> +#include <zencore/timer.h> +#include <zencore/windows.h> +#include <zenstore/cas.h> +#include <zenstore/caslog.h> + +#pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this +#include <rocksdb/db.h> + +#include <lmdb.h> +#include <ppl.h> +#include <spdlog/spdlog.h> +#include <xxh3.h> +#include <asio.hpp> +#include <future> +#include <latch> + +namespace zen { + +namespace rocksdb = ROCKSDB_NAMESPACE; +using namespace fmt::literals; + +////////////////////////////////////////////////////////////////////////// + +struct ProjectStore::OplogStorage : public RefCounted +{ + OplogStorage(ProjectStore::Oplog* OwnerOplog, std::filesystem::path BasePath) : m_OwnerOplog(OwnerOplog), m_OplogStoragePath(BasePath) + { + } + + ~OplogStorage() + { + Log().info("closing oplog storage at {}", m_OplogStoragePath); + Flush(); + + if (m_LmdbEnv) + { + mdb_env_close(m_LmdbEnv); + m_LmdbEnv = nullptr; + } + + if (m_RocksDb) + { + // Column families must be torn down before database is closed + for (const auto& Handle : m_RocksDbColumnHandles) + { + m_RocksDb->DestroyColumnFamilyHandle(Handle); + } + + rocksdb::Status Status = m_RocksDb->Close(); + + if (!Status.ok()) + { + Log().warn("db close error reported for '{}' : '{}'", m_OplogStoragePath, Status.getState()); + } + } + } + + [[nodiscard]] bool Exists() { return Exists(m_OplogStoragePath); } + [[nodiscard]] static bool Exists(std::filesystem::path BasePath) + { + return std::filesystem::exists(BasePath / "ops.zlog") && std::filesystem::exists(BasePath / "ops.zdb") && + std::filesystem::exists(BasePath / "ops.zops"); + } + + static bool Delete(std::filesystem::path BasePath) { return DeleteDirectories(BasePath); } + + void Open(bool IsCreate) + { + Log().info("initializing oplog storage at '{}'", m_OplogStoragePath); + + if (IsCreate) + { + DeleteDirectories(m_OplogStoragePath); + CreateDirectories(m_OplogStoragePath); + } + + m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate); + m_Oplog.Initialize(); + + m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate); + + ZEN_ASSERT(IsPow2(m_OpsAlign)); + ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1))); + + { + std::string LmdbPath = WideToUtf8((m_OplogStoragePath / "ops.zdb").native().c_str()); + + int rc = mdb_env_create(&m_LmdbEnv); + rc = mdb_env_set_mapsize(m_LmdbEnv, 8 * 1024 * 1024); + rc = mdb_env_set_maxreaders(m_LmdbEnv, 256); + rc = mdb_env_open(m_LmdbEnv, LmdbPath.c_str(), MDB_NOSUBDIR | MDB_WRITEMAP | MDB_NOMETASYNC | MDB_NOSYNC, 0666); + } + + { + std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str()); + + Log().debug("opening rocksdb db at '{}'", RocksdbPath); + + rocksdb::DB* Db; + rocksdb::DBOptions Options; + Options.create_if_missing = true; + + std::vector<std::string> ExistingColumnFamilies; + rocksdb::Status Status = rocksdb::DB::ListColumnFamilies(Options, RocksdbPath, &ExistingColumnFamilies); + + std::vector<rocksdb::ColumnFamilyDescriptor> ColumnDescriptors; + + if (Status.IsPathNotFound()) + { + ColumnDescriptors.emplace_back(rocksdb::ColumnFamilyDescriptor{rocksdb::kDefaultColumnFamilyName, {}}); + } + else if (Status.ok()) + { + for (const std::string& Column : ExistingColumnFamilies) + { + rocksdb::ColumnFamilyDescriptor ColumnFamily; + ColumnFamily.name = Column; + ColumnDescriptors.push_back(ColumnFamily); + } + } + else + { + throw std::exception("column family iteration failed for '{}': '{}'"_format(RocksdbPath, Status.getState()).c_str()); + } + + Status = rocksdb::DB::Open(Options, RocksdbPath, ColumnDescriptors, &m_RocksDbColumnHandles, &Db); + + if (!Status.ok()) + { + throw std::exception("database open failed for '{}': '{}'"_format(RocksdbPath, Status.getState()).c_str()); + } + + m_RocksDb.reset(Db); + } + } + + void ReplayLog(std::function<void(CbObject, const OplogEntry&)>&& Handler) + { + // This could use memory mapping or do something clever but for now it just reads the file sequentially + + spdlog::info("replaying log for '{}'", m_OplogStoragePath); + + Stopwatch Timer; + + m_Oplog.Replay([&](const zen::OplogEntry& LogEntry) { + IoBuffer OpBuffer(LogEntry.OpCoreSize); + + const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign; + + m_OpBlobs.Read((void*)OpBuffer.Data(), LogEntry.OpCoreSize, OpFileOffset); + + // Verify checksum, ignore op data if incorrect + const auto OpCoreHash = uint32_t(XXH3_64bits(OpBuffer.Data(), OpBuffer.Size()) & 0xffffFFFF); + + if (OpCoreHash != LogEntry.OpCoreHash) + { + Log().warn("skipping oplog entry with bad checksum!"); + return; + } + + CbObject Op(SharedBuffer::MakeView(OpBuffer.Data(), OpBuffer.Size())); + + m_NextOpsOffset = + Max(m_NextOpsOffset.load(std::memory_order::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); + m_MaxLsn = Max(m_MaxLsn.load(std::memory_order::memory_order_relaxed), LogEntry.OpLsn); + + Handler(Op, LogEntry); + }); + + spdlog::info("Oplog replay completed in {} - Max LSN# {}, Next offset: {}", + NiceTimeSpanMs(Timer.getElapsedTimeMs()), + m_MaxLsn, + m_NextOpsOffset); + } + + OplogEntry AppendOp(CbObject Op) + { + SharedBuffer Buffer = Op.GetBuffer(); + const uint64_t WriteSize = Buffer.GetSize(); + const auto OpCoreHash = uint32_t(XXH3_64bits(Buffer.GetData(), WriteSize) & 0xffffFFFF); + + XXH3_128Stream KeyHasher; + Op["key"].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); }); + XXH3_128 KeyHash = KeyHasher.GetHash(); + + RwLock::ExclusiveLockScope _(m_RwLock); + const uint64_t WriteOffset = m_NextOpsOffset; + const uint32_t OpLsn = ++m_MaxLsn; + + m_NextOpsOffset = RoundUp(WriteOffset + WriteSize, m_OpsAlign); + + ZEN_ASSERT(IsMultipleOf(WriteOffset, m_OpsAlign)); + + OplogEntry Entry = {.OpLsn = OpLsn, + .OpCoreOffset = gsl::narrow_cast<uint32_t>(WriteOffset / m_OpsAlign), + .OpCoreSize = uint32_t(Buffer.GetSize()), + .OpCoreHash = OpCoreHash, + .OpKeyHash = KeyHash}; + + m_Oplog.Append(Entry); + + m_OpBlobs.Write(Buffer.GetData(), WriteSize, WriteOffset); + + return Entry; + } + + void Flush() + { + m_Oplog.Flush(); + m_OpBlobs.Flush(); + } + + spdlog::logger& Log() { return m_OwnerOplog->Log(); } + +private: + ProjectStore::Oplog* m_OwnerOplog; + std::filesystem::path m_OplogStoragePath; + RwLock m_RwLock; + TCasLogFile<OplogEntry> m_Oplog; + CasBlobFile m_OpBlobs; + std::atomic<uint64_t> m_NextOpsOffset{0}; + uint64_t m_OpsAlign = 32; + std::atomic<uint32_t> m_MaxLsn{0}; + MDB_env* m_LmdbEnv = nullptr; + std::unique_ptr<rocksdb::DB> m_RocksDb; + std::vector<rocksdb::ColumnFamilyHandle*> m_RocksDbColumnHandles; +}; + +////////////////////////////////////////////////////////////////////////// + +ProjectStore::Oplog::Oplog(std::string_view Id, Project* Outer, CasStore& Store, std::filesystem::path BasePath) +: m_OuterProject(Outer) +, m_CasStore(Store) +, m_OplogId(Id) +, m_BasePath(BasePath) +{ + m_Storage = new OplogStorage(this, m_BasePath); + const bool StoreExists = m_Storage->Exists(); + m_Storage->Open(/* IsCreate */ !StoreExists); + + m_TempPath = m_BasePath / "temp"; + + zen::CleanDirectory(m_TempPath); +} + +ProjectStore::Oplog::~Oplog() = default; + +bool +ProjectStore::Oplog::ExistsAt(std::filesystem::path BasePath) +{ + return OplogStorage::Exists(BasePath); +} + +void +ProjectStore::Oplog::ReplayLog() +{ + m_Storage->ReplayLog([&](CbObject Op, const OplogEntry& OpEntry) { RegisterOplogEntry(Op, OpEntry, kUpdateReplay); }); +} + +IoBuffer +ProjectStore::Oplog::FindChunk(Oid ChunkId) +{ + if (auto ChunkIt = m_ChunkMap.find(ChunkId); ChunkIt != m_ChunkMap.end()) + { + return m_CasStore.FindChunk(ChunkIt->second); + } + + if (auto FileIt = m_ServerFileMap.find(ChunkId); FileIt != m_ServerFileMap.end()) + { + std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second; + + return IoBufferBuilder::MakeFromFile(FilePath.native().c_str()); + } + + if (auto MetaIt = m_MetaMap.find(ChunkId); MetaIt != m_MetaMap.end()) + { + return m_CasStore.FindChunk(MetaIt->second); + } + + return {}; +} + +void +ProjectStore::Oplog::IterateFileMap(std::function<void(const Oid&, const std::string_view&)>&& Fn) +{ + for (const auto& Kv : m_FileMap) + { + Fn(Kv.first, Kv.second); + } +} + +void +ProjectStore::Oplog::AddFileMapping(Oid FileId, std::string_view Path) +{ + m_FileMap.emplace(FileId, Path); +} + +void +ProjectStore::Oplog::AddServerFileMapping(Oid FileId, std::string_view Path) +{ + m_ServerFileMap.emplace(FileId, Path); +} + +void +ProjectStore::Oplog::AddChunkMapping(Oid ChunkId, IoHash Hash) +{ + m_ChunkMap.emplace(ChunkId, Hash); +} + +void +ProjectStore::Oplog::AddMetaMapping(Oid ChunkId, IoHash Hash) +{ + m_MetaMap.emplace(ChunkId, Hash); +} + +uint32_t +ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry, UpdateType TypeOfUpdate) +{ + ZEN_UNUSED(TypeOfUpdate); + + using namespace std::literals; + + // Update chunk id maps + + if (Core["package"sv]) + { + CbObjectView PkgObj = Core["package"sv].AsObjectView(); + Oid PackageId = PkgObj["id"sv].AsObjectId(); + IoHash PackageHash = PkgObj["data"sv].AsBinaryAttachment(); + + AddChunkMapping(PackageId, PackageHash); + + Log().debug("package data {} -> {}", PackageId, PackageHash); + } + + for (CbFieldView& Entry : Core["bulkdata"sv]) + { + CbObjectView BulkObj = Entry.AsObjectView(); + + Oid BulkDataId = BulkObj["id"sv].AsObjectId(); + IoHash BulkDataHash = BulkObj["data"sv].AsBinaryAttachment(); + + AddChunkMapping(BulkDataId, BulkDataHash); + + Log().debug("bulkdata {} -> {}", BulkDataId, BulkDataHash); + } + + if (CbFieldView FilesArray = Core["files"sv]) + { + int FileCount = 0; + int ServerFileCount = 0; + + std::atomic<bool> InvalidOp{false}; + + Stopwatch Timer; + + std::future<void> f0 = std::async(std::launch::async, [&] { + for (CbFieldView& Entry : FilesArray) + { + CbObjectView FileObj = Entry.AsObjectView(); + const Oid FileId = FileObj["id"sv].AsObjectId(); + + if (auto PathField = FileObj["path"sv]) + { + AddFileMapping(FileId, PathField.AsString()); + + // Log().debug("file {} -> {}", FileId, PathString); + + ++FileCount; + } + else + { + // Every file entry needs to specify a path + InvalidOp = true; + break; + } + + if (InvalidOp.load(std::memory_order::relaxed)) + { + break; + } + } + }); + + std::future<void> f1 = std::async(std::launch::async, [&] { + CbArrayView ServerFilesArray = Core["serverfiles"sv].AsArrayView(); + + for (CbFieldView& Entry : ServerFilesArray) + { + CbObjectView FileObj = Entry.AsObjectView(); + const Oid FileId = FileObj["id"sv].AsObjectId(); + + if (auto PathField = FileObj["path"sv]) + { + AddServerFileMapping(FileId, PathField.AsString()); + + // m_log.debug("file {} -> {}", FileId, PathString); + + ++ServerFileCount; + } + else + { + // Every file entry needs to specify a path + InvalidOp = true; + break; + } + + if (InvalidOp.load(std::memory_order::relaxed)) + { + break; + } + } + }); + + f0.wait(); + f1.wait(); + + if (InvalidOp) + { + return kInvalidOp; + } + + if (FileCount || ServerFileCount) + { + Log().debug("{} files registered, {} server files (took {})", + FileCount, + ServerFileCount, + NiceTimeSpanMs(Timer.getElapsedTimeMs())); + + if (FileCount != ServerFileCount) + { + Log().warn("client/server file list mismatch: {} vs {}", FileCount, ServerFileCount); + } + } + } + + for (CbFieldView& Entry : Core["meta"sv]) + { + CbObjectView MetaObj = Entry.AsObjectView(); + const Oid MetaId = MetaObj["id"sv].AsObjectId(); + auto NameString = MetaObj["name"sv].AsString(); + IoHash MetaDataHash = MetaObj["data"sv].AsBinaryAttachment(); + + AddMetaMapping(MetaId, MetaDataHash); + + Log().debug("meta data ({}) {} -> {}", NameString, MetaId, MetaDataHash); + } + + m_OpAddressMap.emplace(OpEntry.OpLsn, OplogEntryAddress{.Offset = OpEntry.OpCoreOffset, .Size = OpEntry.OpCoreSize}); + m_LatestOpMap[OpEntry.OpKeyAsOId()] = OpEntry.OpLsn; + + return OpEntry.OpLsn; +} + +uint32_t +ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage) +{ + using namespace std::literals; + + const CbObject& Core = OpPackage.GetObject(); + const OplogEntry OpEntry = m_Storage->AppendOp(Core); + + // Persist attachments + + auto Attachments = OpPackage.GetAttachments(); + + for (const auto& Attach : Attachments) + { + SharedBuffer BinaryView = Attach.AsBinaryView(); + m_CasStore.InsertChunk(BinaryView.GetData(), BinaryView.GetSize(), Attach.GetHash()); + } + + return RegisterOplogEntry(Core, OpEntry, kUpdateNewEntry); +} + +////////////////////////////////////////////////////////////////////////// + +ProjectStore::Project::Project(ProjectStore* PrjStore, CasStore& Store, std::filesystem::path BasePath) +: m_ProjectStore(PrjStore) +, m_CasStore(Store) +, m_OplogStoragePath(BasePath) +{ +} + +ProjectStore::Project::~Project() +{ +} + +bool +ProjectStore::Project::Exists(std::filesystem::path BasePath) +{ + return std::filesystem::exists(BasePath / "Project.zcb"); +} + +void +ProjectStore::Project::Read() +{ + std::filesystem::path ProjectStateFilePath = m_OplogStoragePath / "Project.zcb"; + + spdlog::info("reading config for project '{}' from {}", Identifier, ProjectStateFilePath); + + CasBlobFile Blob; + Blob.Open(ProjectStateFilePath, false); + + IoBuffer Obj = Blob.ReadAll(); + CbValidateError ValidationError = ValidateCompactBinary(MemoryView(Obj.Data(), Obj.Size()), CbValidateMode::All); + + if (ValidationError == CbValidateError::None) + { + CbObject Cfg = LoadCompactBinaryObject(Obj); + + Identifier = Cfg["id"].AsString(); + RootDir = Cfg["root"].AsString(); + ProjectRootDir = Cfg["project"].AsString(); + EngineRootDir = Cfg["engine"].AsString(); + } + else + { + spdlog::error("validation error {} hit for '{}'", int(ValidationError), ProjectStateFilePath); + } +} + +void +ProjectStore::Project::Write() +{ + MemoryOutStream Mem; + BinaryWriter Writer(Mem); + + CbObjectWriter Cfg; + Cfg << "id" << Identifier; + Cfg << "root" << WideToUtf8(RootDir.c_str()); + Cfg << "project" << ProjectRootDir; + Cfg << "engine" << EngineRootDir; + + Cfg.Save(Writer); + + CreateDirectories(m_OplogStoragePath); + + std::filesystem::path ProjectStateFilePath = m_OplogStoragePath / "Project.zcb"; + + spdlog::info("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath); + + CasBlobFile Blob; + Blob.Open(ProjectStateFilePath, true); + Blob.Write(Mem.Data(), Mem.Size(), 0); + Blob.Flush(); +} + +spdlog::logger& +ProjectStore::Project::Log() +{ + return m_ProjectStore->Log(); +} + +std::filesystem::path +ProjectStore::Project::BasePathForOplog(std::string_view OplogId) +{ + return m_OplogStoragePath / OplogId; +} + +ProjectStore::Oplog* +ProjectStore::Project::NewOplog(std::string_view OplogId) +{ + RwLock::ExclusiveLockScope _(m_ProjectLock); + + std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); + + try + { + Oplog& Log = m_Oplogs.try_emplace(std::string{OplogId}, OplogId, this, m_CasStore, OplogBasePath).first->second; + + return &Log; + } + catch (std::exception&) + { + // In case of failure we need to ensure there's no half constructed entry around + // + // (This is probably already ensured by the try_emplace implementation?) + + m_Oplogs.erase(std::string{OplogId}); + + return nullptr; + } +} + +ProjectStore::Oplog* +ProjectStore::Project::OpenOplog(std::string_view OplogId) +{ + { + RwLock::SharedLockScope _(m_ProjectLock); + + auto OplogIt = m_Oplogs.find(std::string(OplogId)); + + if (OplogIt != m_Oplogs.end()) + { + return &OplogIt->second; + } + } + + RwLock::ExclusiveLockScope _(m_ProjectLock); + + std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); + + if (Oplog::ExistsAt(OplogBasePath)) + { + // Do open of existing oplog + + try + { + Oplog& Log = m_Oplogs.try_emplace(std::string{OplogId}, OplogId, this, m_CasStore, OplogBasePath).first->second; + + Log.ReplayLog(); + + return &Log; + } + catch (std::exception& ex) + { + spdlog::error("failed to open oplog '{}' @ '{}': {}", OplogId, OplogBasePath, ex.what()); + + m_Oplogs.erase(std::string{OplogId}); + } + } + + return nullptr; +} + +void +ProjectStore::Project::DeleteOplog(std::string_view OplogId) +{ + bool Exists = false; + + { + RwLock::ExclusiveLockScope _(m_ProjectLock); + + auto OplogIt = m_Oplogs.find(std::string(OplogId)); + + if (OplogIt != m_Oplogs.end()) + { + Exists = true; + + m_Oplogs.erase(OplogIt); + } + } + + // Actually erase + + std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); + + OplogStorage::Delete(OplogBasePath); +} + +void +ProjectStore::Project::IterateOplogs(std::function<void(const Oplog&)>&& Fn) const +{ + // TODO: should iterate over oplogs which are present on disk but not yet loaded + + RwLock::SharedLockScope _(m_ProjectLock); + + for (auto& Kv : m_Oplogs) + { + Fn(Kv.second); + } +} + +////////////////////////////////////////////////////////////////////////// + +ProjectStore::ProjectStore(CasStore& Store, std::filesystem::path BasePath) +: m_Log("project", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +, m_ProjectBasePath(BasePath) +, m_CasStore(Store) +{ + m_Log.info("initializing project store at '{}'", BasePath); + m_Log.set_level(spdlog::level::debug); +} + +ProjectStore::~ProjectStore() +{ + m_Log.info("closing project store ('{}')", m_ProjectBasePath); +} + +std::filesystem::path +ProjectStore::BasePathForProject(std::string_view ProjectId) +{ + return m_ProjectBasePath / ProjectId; +} + +ProjectStore::Project* +ProjectStore::OpenProject(std::string_view ProjectId) +{ + { + RwLock::SharedLockScope _(m_ProjectsLock); + + auto ProjIt = m_Projects.find(std::string{ProjectId}); + + if (ProjIt != m_Projects.end()) + { + return &(ProjIt->second); + } + } + + RwLock::ExclusiveLockScope _(m_ProjectsLock); + + std::filesystem::path ProjectBasePath = BasePathForProject(ProjectId); + + if (Project::Exists(ProjectBasePath)) + { + try + { + Log().info("opening project {} @ {}", ProjectId, ProjectBasePath); + + ProjectStore::Project& Prj = m_Projects.try_emplace(std::string{ProjectId}, this, m_CasStore, ProjectBasePath).first->second; + Prj.Read(); + return &Prj; + } + catch (std::exception& e) + { + Log().warn("failed to open {} @ {} ({})", ProjectId, ProjectBasePath, e.what()); + m_Projects.erase(std::string{ProjectId}); + } + } + + return nullptr; +} + +ProjectStore::Project* +ProjectStore::NewProject(std::filesystem::path BasePath, + std::string_view ProjectId, + std::string_view RootDir, + std::string_view EngineRootDir, + std::string_view ProjectRootDir) +{ + RwLock::ExclusiveLockScope _(m_ProjectsLock); + + ProjectStore::Project& Prj = m_Projects.try_emplace(std::string{ProjectId}, this, m_CasStore, BasePath).first->second; + Prj.Identifier = ProjectId; + Prj.RootDir = RootDir; + Prj.EngineRootDir = EngineRootDir; + Prj.ProjectRootDir = ProjectRootDir; + Prj.Write(); + + return &Prj; +} + +void +ProjectStore::DeleteProject(std::string_view ProjectId) +{ + std::filesystem::path ProjectBasePath = BasePathForProject(ProjectId); + + Log().info("deleting project {} @ {}", ProjectId, ProjectBasePath); + + m_Projects.erase(std::string{ProjectId}); + + DeleteDirectories(ProjectBasePath); +} + +bool +ProjectStore::Exists(std::string_view ProjectId) +{ + return Project::Exists(BasePathForProject(ProjectId)); +} + +ProjectStore::Oplog* +ProjectStore::OpenProjectOplog(std::string_view ProjectId, std::string_view OplogId) +{ + if (Project* ProjectIt = OpenProject(ProjectId)) + { + return ProjectIt->OpenOplog(OplogId); + } + + return nullptr; +} + +////////////////////////////////////////////////////////////////////////// + +HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) +: m_CasStore(Store) +, m_Log("project", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +, m_ProjectStore(Projects) +{ + using namespace std::literals; + + m_Router.AddPattern("project", "([[:alnum:]_.]+)"); + m_Router.AddPattern("log", "([[:alnum:]_.]+)"); + m_Router.AddPattern("op", "([[:digit:]]+?)"); + m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/batch", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + m_Log.info("batch - {} / {}", ProjectId, OplogId); + + ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId); + + if (FoundLog == nullptr) + { + return HttpReq.WriteResponse(HttpResponse::NotFound); + } + + ProjectStore::Oplog& Log = *FoundLog; + + // Parse Request + + IoBuffer Payload = HttpReq.ReadPayload(); + MemoryInStream MemIn(Payload.Data(), Payload.Size()); + BinaryReader Reader(MemIn); + + struct RequestHeader + { + enum + { + kMagic = 0xAAAA'77AC + }; + uint32_t Magic; + uint32_t ChunkCount; + uint32_t Reserved1; + uint32_t Reserved2; + }; + + struct RequestChunkEntry + { + Oid ChunkId; + uint32_t CorrelationId; + uint64_t Offset; + uint64_t RequestBytes; + }; + + if (Payload.Size() <= sizeof(RequestHeader)) + { + HttpReq.WriteResponse(HttpResponse::BadRequest); + } + + RequestHeader Hdr; + Reader.Read(&Hdr, sizeof Hdr); + + if (Hdr.Magic != RequestHeader::kMagic) + { + HttpReq.WriteResponse(HttpResponse::BadRequest); + } + + // Make Response + + MemoryOutStream MemOut; + BinaryWriter MemWriter(MemOut); + + struct ResponseHeader + { + uint32_t Magic = 0xbada'b00f; + uint32_t ChunkCount; + uint32_t Reserved1 = 0; + uint32_t Reserved2 = 0; + }; + + struct ResponseChunkEntry + { + uint32_t CorrelationId; + uint32_t Flags = 0; + uint64_t ChunkSize; + }; + + return HttpReq.WriteResponse(HttpResponse::NotFound); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/files", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + // File manifest fetch, returns the client file list + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId); + + if (FoundLog == nullptr) + { + return HttpReq.WriteResponse(HttpResponse::NotFound); + } + + ProjectStore::Oplog& Log = *FoundLog; + + CbObjectWriter Response; + Response.BeginArray("files"); + + Log.IterateFileMap([&](const Oid& Id, const std::string_view& Path) { + Response.BeginObject(); + Response << "id" << Id; + Response << "path" << Path; + Response.EndObject(); + }); + + Response.EndArray(); + + return HttpReq.WriteResponse(HttpResponse::OK, Response.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{chunk}/info", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& ChunkId = Req.GetCapture(3); + + ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId); + + if (FoundLog == nullptr) + { + return HttpReq.WriteResponse(HttpResponse::NotFound); + } + + ProjectStore::Oplog& Log = *FoundLog; + + Oid Obj = Oid::FromHexString(ChunkId); + + IoBuffer Value = Log.FindChunk(Obj); + + if (Value) + { + CbObjectWriter Response; + Response << "size" << Value.Size(); + return HttpReq.WriteResponse(HttpResponse::OK, Response.Save()); + } + + return HttpReq.WriteResponse(HttpResponse::NotFound); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{chunk}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& ChunkId = Req.GetCapture(3); + + bool IsOffset = false; + uint64_t Offset = 0; + uint64_t Size = ~(0ull); + + auto QueryParms = Req.ServerRequest().GetQueryParams(); + + if (auto OffsetParm = QueryParms.GetValue("offset"); OffsetParm.empty() == false) + { + if (auto OffsetVal = ParseInt<uint64_t>(OffsetParm)) + { + Offset = OffsetVal.value(); + IsOffset = true; + } + else + { + return HttpReq.WriteResponse(HttpResponse::BadRequest); + } + } + + if (auto SizeParm = QueryParms.GetValue("size"); SizeParm.empty() == false) + { + if (auto SizeVal = ParseInt<uint64_t>(SizeParm)) + { + Size = SizeVal.value(); + IsOffset = true; + } + else + { + return HttpReq.WriteResponse(HttpResponse::BadRequest); + } + } + + m_Log.debug("chunk - {} / {} / {}", ProjectId, OplogId, ChunkId); + + ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId); + + if (FoundLog == nullptr) + { + return HttpReq.WriteResponse(HttpResponse::NotFound); + } + + ProjectStore::Oplog& Log = *FoundLog; + + Oid Obj = Oid::FromHexString(ChunkId); + + IoBuffer Value = Log.FindChunk(Obj); + + switch (HttpVerb Verb = HttpReq.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + if (!Value) + { + return HttpReq.WriteResponse(HttpResponse::NotFound); + } + + if (Verb == HttpVerb::kHead) + { + HttpReq.SetSuppressResponseBody(); + } + + if (IsOffset) + { + if (Offset > Value.Size()) + { + Offset = Value.Size(); + } + + if ((Offset + Size) > Value.Size()) + { + Size = Value.Size() - Offset; + } + + // Send only a subset of data + IoBuffer InnerValue(Value, Offset, Size); + + return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, InnerValue); + } + + return HttpReq.WriteResponse(HttpResponse::OK, HttpContentType::kBinary, Value); + } + }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/new", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId); + + if (FoundLog == nullptr) + { + return HttpReq.WriteResponse(HttpResponse::NotFound); + } + + ProjectStore::Oplog& Log = *FoundLog; + + IoBuffer Payload = HttpReq.ReadPayload(); + + CbPackage Package; + Package.Load(Payload); + + CbObject Core = Package.GetObject(); + + if (!Core["key"sv]) + { + return HttpReq.WriteResponse(HttpResponse::BadRequest, HttpContentType::kText, "No oplog entry key specified"); + } + + // Write core to oplog + + const uint32_t OpLsn = Log.AppendNewOplogEntry(Package); + + if (OpLsn == ProjectStore::Oplog::kInvalidOp) + { + return HttpReq.WriteResponse(HttpResponse::BadRequest); + } + + m_Log.info("new op #{:4} - {}/{} ({:>6}) {}", OpLsn, ProjectId, OplogId, NiceBytes(Payload.Size()), Core["key"sv].AsString()); + + HttpReq.WriteResponse(HttpResponse::Created); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{op}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + // TODO: look up op and respond with the payload! + + HttpReq.WriteResponse(HttpResponse::Accepted, HttpContentType::kText, u8"yeee"sv); + }, + HttpVerb::kGet); + + using namespace fmt::literals; + + m_Router.RegisterRoute( + "{project}/oplog/{log}", + [this](HttpRouterRequest& Req) { + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + ProjectStore::Project* ProjectIt = m_ProjectStore->OpenProject(ProjectId); + + if (!ProjectIt) + { + return Req.ServerRequest().WriteResponse(HttpResponse::NotFound, + HttpContentType::kText, + "project {} not found"_format(ProjectId)); + } + + ProjectStore::Project& Prj = *ProjectIt; + + switch (Req.ServerRequest().RequestVerb()) + { + case HttpVerb::kGet: + { + ProjectStore::Oplog* OplogIt = Prj.OpenOplog(OplogId); + + if (!OplogIt) + { + return Req.ServerRequest().WriteResponse(HttpResponse::NotFound, + HttpContentType::kText, + "oplog {} not found in project {}"_format(OplogId, ProjectId)); + } + + ProjectStore::Oplog& Log = *OplogIt; + + CbObjectWriter Cb; + Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempDir(); + + Req.ServerRequest().WriteResponse(HttpResponse::OK, Cb.Save()); + } + break; + + case HttpVerb::kPost: + { + ProjectStore::Oplog* OplogIt = Prj.OpenOplog(OplogId); + + if (!OplogIt) + { + if (!Prj.NewOplog(OplogId)) + { + // TODO: indicate why the operation failed! + return Req.ServerRequest().WriteResponse(HttpResponse::InternalServerError); + } + + m_Log.info("established oplog {} / {}", ProjectId, OplogId); + + return Req.ServerRequest().WriteResponse(HttpResponse::Created); + } + + // I guess this should ultimately be used to execute RPCs but for now, it + // does absolutely nothing + + return Req.ServerRequest().WriteResponse(HttpResponse::BadRequest); + } + break; + + case HttpVerb::kDelete: + { + spdlog::info("deleting oplog {}/{}", ProjectId, OplogId); + + ProjectIt->DeleteOplog(OplogId); + + return Req.ServerRequest().WriteResponse(HttpResponse::OK); + } + break; + } + }, + HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{project}", + [this](HttpRouterRequest& Req) { + const std::string ProjectId = Req.GetCapture(1); + + switch (Req.ServerRequest().RequestVerb()) + { + case HttpVerb::kPost: + { + IoBuffer Payload = Req.ServerRequest().ReadPayload(); + CbObject Params = LoadCompactBinaryObject(Payload); + std::string_view Id = Params["id"sv].AsString(); + std::string_view Root = Params["root"sv].AsString(); + std::string_view EngineRoot = Params["engine"sv].AsString(); + std::string_view ProjectRoot = Params["project"sv].AsString(); + + const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; + m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot); + + m_Log.info("established project - {} (id: '{}', roots: '{}', '{}', '{}')", + ProjectId, + Id, + Root, + EngineRoot, + ProjectRoot); + + Req.ServerRequest().WriteResponse(HttpResponse::Created); + } + break; + + case HttpVerb::kGet: + { + ProjectStore::Project* ProjectIt = m_ProjectStore->OpenProject(ProjectId); + + if (!ProjectIt) + { + return Req.ServerRequest().WriteResponse(HttpResponse::NotFound, + HttpContentType::kText, + "project {} not found"_format(ProjectId)); + } + + const ProjectStore::Project& Prj = *ProjectIt; + + CbObjectWriter Response; + Response << "id" << Prj.Identifier << "root" << WideToUtf8(Prj.RootDir.c_str()); + + Response.BeginArray("oplogs"sv); + Prj.IterateOplogs([&](const ProjectStore::Oplog& I) { Response << "id"sv << I.OplogId(); }); + Response.EndArray(); // oplogs + + Req.ServerRequest().WriteResponse(HttpResponse::OK, Response.Save()); + } + break; + + case HttpVerb::kDelete: + { + ProjectStore::Project* ProjectIt = m_ProjectStore->OpenProject(ProjectId); + + if (!ProjectIt) + { + return Req.ServerRequest().WriteResponse(HttpResponse::NotFound, + HttpContentType::kText, + "project {} not found"_format(ProjectId)); + } + + m_ProjectStore->DeleteProject(ProjectId); + } + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete); +} + +HttpProjectService::~HttpProjectService() +{ +} + +const char* +HttpProjectService::BaseUri() const +{ + return "/prj/"; +} + +void +HttpProjectService::HandleRequest(HttpServerRequest& Request) +{ + if (m_Router.HandleRequest(Request) == false) + { + m_Log.warn("No route found for {0}", Request.RelativeUri()); + } +} + +////////////////////////////////////////////////////////////////////////// + +class SecurityAttributes +{ +public: + inline SECURITY_ATTRIBUTES* Attributes() { return &m_Attributes; } + +protected: + SECURITY_ATTRIBUTES m_Attributes{}; + SECURITY_DESCRIPTOR m_Sd{}; +}; + +// Security attributes which allows any user access + +class AnyUserSecurityAttributes : public SecurityAttributes +{ +public: + AnyUserSecurityAttributes() + { + m_Attributes.nLength = sizeof m_Attributes; + m_Attributes.bInheritHandle = false; // Disable inheritance + + const BOOL success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION); + + if (success) + { + const BOOL bSetOk = SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE); + if (bSetOk) + { + m_Attributes.lpSecurityDescriptor = &m_Sd; + } + } + } +}; + +////////////////////////////////////////////////////////////////////////// + +struct LocalProjectService::LocalProjectImpl +{ + LocalProjectImpl() : m_WorkerThreadPool(ServiceThreadCount) {} + ~LocalProjectImpl() { Stop(); } + + void Start() + { + ZEN_ASSERT(!m_IsStarted); + + for (int i = 0; i < 32; ++i) + { + PipeConnection* NewPipe = new PipeConnection(this); + m_ServicePipes.push_back(NewPipe); + m_IoContext.post([NewPipe] { NewPipe->Accept(); }); + } + + for (int i = 0; i < ServiceThreadCount; ++i) + { + asio::post(m_WorkerThreadPool, [this] { + try + { + m_IoContext.run(); + } + catch (std::exception& ex) + { + spdlog::error("exception caught in pipe project service loop: {}", ex.what()); + } + + m_ShutdownLatch.count_down(); + }); + } + + m_IsStarted = true; + } + + void Stop() + { + if (!m_IsStarted) + { + return; + } + + for (PipeConnection* Pipe : m_ServicePipes) + { + Pipe->Disconnect(); + } + + m_IoContext.stop(); + m_ShutdownLatch.wait(); + + for (PipeConnection* Pipe : m_ServicePipes) + { + delete Pipe; + } + + m_ServicePipes.clear(); + } + +private: + asio::io_context& IoContext() { return m_IoContext; } + auto PipeSecurityAttributes() { return m_AnyUserSecurityAttributes.Attributes(); } + static const int ServiceThreadCount = 4; + + std::latch m_ShutdownLatch{ServiceThreadCount}; + asio::thread_pool m_WorkerThreadPool; + asio::io_context m_IoContext; + + class PipeConnection + { + enum PipeState + { + kUninitialized, + kConnecting, + kReading, + kWriting, + kDisconnected, + kInvalid + }; + + LocalProjectImpl* m_Outer; + asio::windows::stream_handle m_PipeHandle; + std::atomic<PipeState> m_PipeState{kUninitialized}; + + public: + PipeConnection(LocalProjectImpl* Outer) : m_Outer(Outer), m_PipeHandle{m_Outer->IoContext()} {} + ~PipeConnection() {} + + void Disconnect() + { + m_PipeState = kDisconnected; + DisconnectNamedPipe(m_PipeHandle.native_handle()); + } + + void Accept() + { + StringBuilder<64> PipeName; + PipeName << "\\\\.\\pipe\\zenprj"; // TODO: this should use an instance-specific identifier! + + HANDLE hPipe = CreateNamedPipeA(PipeName.c_str(), + PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED, + PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT, + PIPE_UNLIMITED_INSTANCES, // Max instance count + 65536, // Output buffer size + 65536, // Input buffer size + 10'000, // Default timeout (ms) + m_Outer->PipeSecurityAttributes() // Security attributes + ); + + if (hPipe == INVALID_HANDLE_VALUE) + { + spdlog::warn("failed while creating named pipe {}", PipeName.c_str()); + + // TODO: error - how to best handle? + } + + m_PipeHandle.assign(hPipe); // This now owns the handle and will close it + + m_PipeState = kConnecting; + + asio::windows::overlapped_ptr OverlappedPtr( + m_PipeHandle.get_executor(), + std::bind(&PipeConnection::OnClientConnect, this, std::placeholders::_1, std::placeholders::_2)); + + OVERLAPPED* Overlapped = OverlappedPtr.get(); + BOOL Ok = ConnectNamedPipe(hPipe, Overlapped); + DWORD LastError = GetLastError(); + + if (!Ok && LastError != ERROR_IO_PENDING) + { + m_PipeState = kInvalid; + + // The operation completed immediately, so a completion notification needs + // to be posted. When complete() is called, ownership of the OVERLAPPED- + // derived object passes to the io_service. + std::error_code Ec(LastError, asio::error::get_system_category()); + OverlappedPtr.complete(Ec, 0); + } + else + { + // The operation was successfully initiated, so ownership of the + // OVERLAPPED-derived object has now passed to the io_service. + OverlappedPtr.release(); + } + } + + private: + void OnClientConnect(const std::error_code& Ec, size_t BytesTransferred) + { + ZEN_UNUSED(BytesTransferred); + + if (Ec) + { + if (m_PipeState == kDisconnected) + { + return; + } + + spdlog::warn("pipe connection error: {}", Ec.message()); + + // TODO: should disconnect and issue a new connect + return; + } + + spdlog::debug("pipe connection established"); + + IssueRead(); + } + + void IssueRead() + { + m_PipeState = kReading; + + m_PipeHandle.async_read_some(asio::mutable_buffer(m_MsgBuffer, sizeof m_MsgBuffer), + std::bind(&PipeConnection::OnClientRead, this, std::placeholders::_1, std::placeholders::_2)); + } + + void OnClientRead(const std::error_code& Ec, size_t Bytes) + { + if (Ec) + { + if (m_PipeState == kDisconnected) + { + return; + } + + spdlog::warn("pipe read error: {}", Ec.message()); + + // TODO: should disconnect and issue a new connect + return; + } + + spdlog::debug("received message: {} bytes", Bytes); + + // TODO: Actually process request + + m_PipeState = kWriting; + + asio::async_write(m_PipeHandle, + asio::buffer(m_MsgBuffer, Bytes), + std::bind(&PipeConnection::OnWriteCompletion, this, std::placeholders::_1, std::placeholders::_2)); + } + + void OnWriteCompletion(const std::error_code& Ec, size_t Bytes) + { + ZEN_UNUSED(Bytes); + + if (Ec) + { + if (m_PipeState == kDisconnected) + { + return; + } + + spdlog::warn("pipe write error: {}", Ec.message()); + + // TODO: should disconnect and issue a new connect + return; + } + + // Go back to reading + IssueRead(); + } + + uint8_t m_MsgBuffer[16384]; + }; + + AnyUserSecurityAttributes m_AnyUserSecurityAttributes; + std::vector<PipeConnection*> m_ServicePipes; + bool m_IsStarted = false; +}; + +LocalProjectService::LocalProjectService(CasStore& Store, ProjectStore* Projects) : m_CasStore(Store), m_ProjectStore(Projects) +{ + m_Impl = std::make_unique<LocalProjectImpl>(); + m_Impl->Start(); +} + +LocalProjectService::~LocalProjectService() +{ + m_Impl->Stop(); +} + +////////////////////////////////////////////////////////////////////////// + +} // namespace zen diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h new file mode 100644 index 000000000..4ad0e42e0 --- /dev/null +++ b/zenserver/projectstore.h @@ -0,0 +1,241 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/httpserver.h> +#include <zencore/uid.h> +#include <zencore/xxhash.h> +#include <zenstore/cas.h> +#include <zenstore/caslog.h> + +#include <spdlog/spdlog.h> +#include <tsl/robin_map.h> +#include <filesystem> +#include <map> +#include <string> + +namespace zen { + +class CbPackage; + +struct OplogEntry +{ + uint32_t OpLsn; + uint32_t OpCoreOffset; // note: Multiple of alignment! + uint32_t OpCoreSize; + uint32_t OpCoreHash; // Used as checksum + XXH3_128 OpKeyHash; // XXH128_canonical_t + + inline Oid OpKeyAsOId() const + { + Oid Id; + memcpy(Id.OidBits, &OpKeyHash, sizeof Id.OidBits); + return Id; + } +}; + +static_assert(IsPow2(sizeof(OplogEntry))); + +/** Project Store + */ +class ProjectStore : public RefCounted +{ + struct OplogStorage; + +public: + ProjectStore(CasStore& Store, std::filesystem::path BasePath); + ~ProjectStore(); + + struct Project; + + struct Oplog + { + Oplog(std::string_view Id, Project* Outer, CasStore& Store, std::filesystem::path BasePath); + ~Oplog(); + + [[nodiscard]] static bool ExistsAt(std::filesystem::path BasePath); + + void IterateFileMap(std::function<void(const Oid&, const std::string_view&)>&& Fn); + + IoBuffer FindChunk(Oid ChunkId); + + inline static const uint32_t kInvalidOp = ~0u; + + /** Persist a new oplog entry + * + * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected + */ + uint32_t AppendNewOplogEntry(CbPackage Op); + + enum UpdateType + { + kUpdateNewEntry, + kUpdateReplay + }; + + /** Update tracking metadata for a new oplog entry + * + * This is used during replay (and gets called as part of new op append) + * + * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected + */ + uint32_t RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry, UpdateType TypeOfUpdate); + + /** Scan oplog and register each entry, thus updating the in-memory tracking tables + */ + void ReplayLog(); + + const std::string& OplogId() const { return m_OplogId; } + + const std::wstring& TempDir() const { return m_TempPath.native(); } + + spdlog::logger& Log() { return m_OuterProject->Log(); } + + private: + struct OplogEntryAddress + { + uint64_t Offset; + uint64_t Size; + }; + + template<class V> + using OidMap = tsl::robin_map<Oid, V, Oid::Hasher>; + + Project* m_OuterProject = nullptr; + RwLock m_OplogLock; + CasStore& m_CasStore; + std::filesystem::path m_BasePath; + std::filesystem::path m_TempPath; + + OidMap<IoHash> m_ChunkMap; // output data chunk id -> CAS address + OidMap<IoHash> m_MetaMap; // meta chunk id -> CAS address + OidMap<std::string> m_FileMap; // file id -> client file + OidMap<std::string> m_ServerFileMap; // file id -> server file + std::map<int, OplogEntryAddress> m_OpAddressMap; // Index LSN -> op data in ops blob file + OidMap<int> m_LatestOpMap; // op key -> latest op LSN for key + + RefPtr<OplogStorage> m_Storage; + std::string m_OplogId; + + void AddFileMapping(Oid FileId, std::string_view Path); + void AddServerFileMapping(Oid FileId, std::string_view Path); + void AddChunkMapping(Oid ChunkId, IoHash Hash); + void AddMetaMapping(Oid ChunkId, IoHash Hash); + }; + + struct Project + { + std::string Identifier; + std::filesystem::path RootDir; + std::string EngineRootDir; + std::string ProjectRootDir; + + Oplog* NewOplog(std::string_view OplogId); + Oplog* OpenOplog(std::string_view OplogId); + void DeleteOplog(std::string_view OplogId); + void IterateOplogs(std::function<void(const Oplog&)>&& Fn) const; + + Project(ProjectStore* PrjStore, CasStore& Store, std::filesystem::path BasePath); + ~Project(); + + void Read(); + void Write(); + [[nodiscard]] static bool Exists(std::filesystem::path BasePath); + + spdlog::logger& Log(); + + private: + ProjectStore* m_ProjectStore; + CasStore& m_CasStore; + mutable RwLock m_ProjectLock; + std::map<std::string, Oplog> m_Oplogs; + std::filesystem::path m_OplogStoragePath; + + std::filesystem::path BasePathForOplog(std::string_view OplogId); + }; + + Oplog* OpenProjectOplog(std::string_view ProjectId, std::string_view OplogId); + + Project* OpenProject(std::string_view ProjectId); + Project* NewProject(std::filesystem::path BasePath, + std::string_view ProjectId, + std::string_view RootDir, + std::string_view EngineRootDir, + std::string_view ProjectRootDir); + void DeleteProject(std::string_view ProjectId); + bool Exists(std::string_view ProjectId); + + spdlog::logger& Log() { return m_Log; } + const std::filesystem::path& BasePath() const { return m_ProjectBasePath; } + +private: + spdlog::logger m_Log; + CasStore& m_CasStore; + std::filesystem::path m_ProjectBasePath; + RwLock m_ProjectsLock; + std::map<std::string, Project> m_Projects; + + std::filesystem::path BasePathForProject(std::string_view ProjectId); +}; + +////////////////////////////////////////////////////////////////////////// +// +// {ns} a root namespace, should be associated with the project which owns it +// {target} a variation of the project, typically a build target +// {lsn} oplog entry sequence number +// +// /prj/{ns} +// /prj/{ns}/oplog/{target} +// /prj/{ns}/oplog/{target}/{lsn} +// +// oplog entry +// +// id: {id} +// key: {} +// meta: {} +// data: [] +// refs: +// + +class HttpProjectService : public HttpService +{ +public: + HttpProjectService(CasStore& Store, ProjectStore* Projects); + ~HttpProjectService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + +private: + CasStore& m_CasStore; + spdlog::logger m_Log; + HttpRequestRouter m_Router; + Ref<ProjectStore> m_ProjectStore; +}; + +/** Project store interface for local clients + * + * This provides the same functionality as the HTTP interface but with + * some optimizations which are only possible for clients running on the + * same host as the Zen Store instance + * + */ + +class LocalProjectService : public RefCounted +{ +protected: + LocalProjectService(CasStore& Store, ProjectStore* Projects); + ~LocalProjectService(); + +public: + static inline Ref<LocalProjectService> New(CasStore& Store, ProjectStore* Projects) { return new LocalProjectService(Store, Projects); } + +private: + struct LocalProjectImpl; + + CasStore& m_CasStore; + Ref<ProjectStore> m_ProjectStore; + std::unique_ptr<LocalProjectImpl> m_Impl; +}; + +} // namespace zen diff --git a/zenserver/targetver.h b/zenserver/targetver.h new file mode 100644 index 000000000..d432d6993 --- /dev/null +++ b/zenserver/targetver.h @@ -0,0 +1,10 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +// Including SDKDDKVer.h defines the highest available Windows platform. + +// If you wish to build your application for a previous Windows platform, include WinSDKVer.h and +// set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. + +#include <SDKDDKVer.h> diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp new file mode 100644 index 000000000..119055e44 --- /dev/null +++ b/zenserver/testing/launch.cpp @@ -0,0 +1,490 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "launch.h" + +#include <zencore/compactbinary.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/windows.h> +#include <zenstore/CAS.h> + +#include <AccCtrl.h> +#include <AclAPI.h> +#include <sddl.h> + +#include <UserEnv.h> +#pragma comment(lib, "UserEnv.lib") + +#include <atlbase.h> +#include <filesystem> +#include <span> + +using namespace std::literals; + +namespace zen { + +struct BasicJob +{ +public: + BasicJob() = default; + ~BasicJob(); + + void SetWorkingDirectory(const std::filesystem::path& WorkingDirectory) { m_WorkingDirectory = WorkingDirectory; } + bool SpawnJob(std::filesystem::path ExePath, std::wstring CommandLine); + bool Wait(uint32_t TimeoutMs = ~0); + +private: + std::filesystem::path m_WorkingDirectory; + int m_ProcessId = 0; + CHandle m_ProcessHandle; +}; + +BasicJob::~BasicJob() +{ + Wait(); +} + +bool +BasicJob::SpawnJob(std::filesystem::path ExePath, std::wstring CommandLine) +{ + using namespace fmt::literals; + + STARTUPINFOEX StartupInfo = {sizeof(STARTUPINFOEX)}; + PROCESS_INFORMATION ProcessInfo{}; + + std::wstring ExePathNative = ExePath.native(); + std::wstring WorkingDirNative = m_WorkingDirectory.native(); + + BOOL Created = ::CreateProcess(ExePathNative.data() /* ApplicationName */, + CommandLine.data() /* Command Line */, + nullptr /* Process Attributes */, + nullptr /* Security Attributes */, + FALSE /* InheritHandles */, + 0 /* Flags */, + nullptr /* Environment */, + WorkingDirNative.data() /* Current Directory */, + (LPSTARTUPINFO)&StartupInfo, + &ProcessInfo); + + if (!Created) + { + throw std::system_error(::GetLastError(), std::system_category(), "Failed to create process '{}'"_format(ExePath).c_str()); + } + + m_ProcessId = ProcessInfo.dwProcessId; + m_ProcessHandle.Attach(ProcessInfo.hProcess); + ::CloseHandle(ProcessInfo.hThread); + + spdlog::info("Created process {}", m_ProcessId); + + return true; +} + +bool +BasicJob::Wait(uint32_t TimeoutMs) +{ + if (!m_ProcessHandle) + { + return true; + } + + DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, TimeoutMs); + + if (WaitResult == WAIT_TIMEOUT) + { + return false; + } + + if (WaitResult == WAIT_OBJECT_0) + { + return true; + } + + throw std::exception("Failed wait on process handle"); +} + +struct SandboxedJob +{ + SandboxedJob() = default; + ~SandboxedJob() = default; + + void SetWorkingDirectory(const std::filesystem::path& WorkingDirectory) { m_WorkingDirectory = WorkingDirectory; } + void Initialize(std::string_view AppContainerId); + bool SpawnJob(std::filesystem::path ExePath); + void AddWhitelistFile(const std::filesystem::path& FilePath) { m_WhitelistFiles.push_back(FilePath); } + +private: + bool GrantNamedObjectAccess(PWSTR Name, SE_OBJECT_TYPE Type, ACCESS_MASK AccessMask, bool Recursive); + + std::filesystem::path m_WorkingDirectory; + std::vector<std::filesystem::path> m_WhitelistFiles; + std::vector<std::wstring> m_WhitelistRegistryKeys; + PSID m_AppContainerSid = nullptr; + bool m_IsInitialized = false; +}; + +bool +SandboxedJob::GrantNamedObjectAccess(PWSTR ObjectName, SE_OBJECT_TYPE ObjectType, ACCESS_MASK AccessMask, bool Recursive) +{ + DWORD Status; + PACL NewAcl = nullptr; + + DWORD grfInhericance = 0; + + if (Recursive) + { + grfInhericance = OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE; + } + + EXPLICIT_ACCESS Access{.grfAccessPermissions = AccessMask, + .grfAccessMode = GRANT_ACCESS, + .grfInheritance = grfInhericance, + .Trustee = {.pMultipleTrustee = nullptr, + .MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE, + .TrusteeForm = TRUSTEE_IS_SID, + .TrusteeType = TRUSTEE_IS_GROUP, + .ptstrName = (PWSTR)m_AppContainerSid}}; + + PACL OldAcl = nullptr; + + Status = GetNamedSecurityInfo(ObjectName /* ObjectName */, + ObjectType /* ObjectType */, + DACL_SECURITY_INFORMATION /* SecurityInfo */, + nullptr /* ppsidOwner */, + nullptr /* ppsidGroup */, + &OldAcl /* ppDacl */, + nullptr /* ppSacl */, + nullptr /* ppSecurityDescriptor */); + if (Status != ERROR_SUCCESS) + return false; + + Status = SetEntriesInAcl(1 /* CountOfExplicitEntries */, &Access /* pListOfExplicitEntries */, OldAcl, &NewAcl); + if (Status != ERROR_SUCCESS) + return false; + + Status = SetNamedSecurityInfo(ObjectName /* ObjectName */, + ObjectType /* ObjectType */, + DACL_SECURITY_INFORMATION /*SecurityInfo */, + nullptr /* psidOwner */, + nullptr /* psidGroup */, + NewAcl /* pDacl */, + nullptr /* pSacl */); + if (NewAcl) + ::LocalFree(NewAcl); + + return Status == ERROR_SUCCESS; +} + +void +SandboxedJob::Initialize(std::string_view AppContainerId) +{ + if (m_IsInitialized) + { + return; + } + + std::wstring ContainerName = zen::Utf8ToWide(AppContainerId); + + HRESULT hRes = ::CreateAppContainerProfile(ContainerName.c_str(), + ContainerName.c_str() /* Display Name */, + ContainerName.c_str() /* Description */, + nullptr /* Capabilities */, + 0 /* Capability Count */, + &m_AppContainerSid); + + if (FAILED(hRes)) + { + hRes = ::DeriveAppContainerSidFromAppContainerName(ContainerName.c_str(), &m_AppContainerSid); + + if (FAILED(hRes)) + { + spdlog::error("Failed creating app container SID"); + } + } + + // Debugging context + + PWSTR Str = nullptr; + ::ConvertSidToStringSid(m_AppContainerSid, &Str); + + spdlog::info("AppContainer SID : '{}'", WideToUtf8(Str)); + + PWSTR Path = nullptr; + if (SUCCEEDED(::GetAppContainerFolderPath(Str, &Path))) + { + spdlog::info("AppContainer folder: '{}'", WideToUtf8(Path)); + + ::CoTaskMemFree(Path); + } + ::LocalFree(Str); + + m_IsInitialized = true; +} + +bool +SandboxedJob::SpawnJob(std::filesystem::path ExePath) +{ + // Build process attributes + + SECURITY_CAPABILITIES Sc = {0}; + Sc.AppContainerSid = m_AppContainerSid; + + STARTUPINFOEX StartupInfo = {sizeof(STARTUPINFOEX)}; + PROCESS_INFORMATION ProcessInfo{}; + SIZE_T Size = 0; + + ::InitializeProcThreadAttributeList(nullptr, 1, 0, &Size); + + auto AttrBuffer = std::make_unique<uint8_t[]>(Size); + StartupInfo.lpAttributeList = reinterpret_cast<LPPROC_THREAD_ATTRIBUTE_LIST>(AttrBuffer.get()); + + if (!::InitializeProcThreadAttributeList(StartupInfo.lpAttributeList, 1, 0, &Size)) + { + return false; + } + + if (!::UpdateProcThreadAttribute(StartupInfo.lpAttributeList, + 0, + PROC_THREAD_ATTRIBUTE_SECURITY_CAPABILITIES, + &Sc, + sizeof Sc, + nullptr, + nullptr)) + { + return false; + } + + // Set up security for files/folders/registry + + for (const std::filesystem::path& File : m_WhitelistFiles) + { + std::wstring NativeFileName = File.native(); + GrantNamedObjectAccess(NativeFileName.data(), SE_FILE_OBJECT, FILE_ALL_ACCESS, true); + } + + for (std::wstring& RegKey : m_WhitelistRegistryKeys) + { + GrantNamedObjectAccess(RegKey.data(), SE_REGISTRY_WOW64_32KEY, KEY_ALL_ACCESS, true); + } + + std::wstring ExePathNative = ExePath.native(); + std::wstring WorkingDirNative = m_WorkingDirectory.native(); + + BOOL Created = ::CreateProcess(nullptr /* ApplicationName */, + ExePathNative.data() /* Command line */, + nullptr /* Process Attributes */, + nullptr /* Security Attributes */, + FALSE /* InheritHandles */, + EXTENDED_STARTUPINFO_PRESENT | CREATE_NEW_CONSOLE /* Flags */, + nullptr /* Environment */, + WorkingDirNative.data() /* Current Directory */, + (LPSTARTUPINFO)&StartupInfo, + &ProcessInfo); + + DeleteProcThreadAttributeList(StartupInfo.lpAttributeList); + + if (!Created) + { + return false; + } + + spdlog::info("Created process {}", ProcessInfo.dwProcessId); + + return true; +} + +HttpLaunchService::HttpLaunchService(CasStore& Store) +: m_Log("exec", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +, m_CasStore(Store) +{ + m_Router.AddPattern("job", "([[:digit:]]+)"); + + m_Router.RegisterRoute( + "jobs/{job}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + break; + + case HttpVerb::kPost: + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost); + + // Experimental + +#if 0 + m_Router.RegisterRoute( + "jobs/sandbox", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + break; + + case HttpVerb::kPost: + { + SandboxedJob Job; + Job.Initialize("zen_test"); + Job.SetWorkingDirectory("c:\\temp\\sandbox1"); + Job.AddWhitelistFile("c:\\temp\\sandbox1"); + Job.SpawnJob("c:\\windows\\system32\\cmd.exe"); + } + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost); +#endif + + m_Router.RegisterRoute( + "jobs/prep", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kPost: + { + // This operation takes the proposed job spec and identifies which + // chunks are not present on this server. This list is then returned in + // the "need" list in the response + + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject RequestObject = LoadCompactBinaryObject(Payload); + + std::vector<IoHash> NeedList; + + for (auto Entry : RequestObject["files"sv]) + { + CbObjectView Ob = Entry.AsObjectView(); + + const IoHash FileHash = Ob["hash"sv].AsHash(); + + if (!m_CasStore.FindChunk(FileHash)) + { + spdlog::debug("NEED: {} {} {}", FileHash, Ob["file"sv].AsString(), Ob["size"sv].AsUInt64()); + + NeedList.push_back(FileHash); + } + } + + CbObjectWriter Cbo; + Cbo.BeginArray("need"); + + for (const IoHash& Hash : NeedList) + { + Cbo << Hash; + } + + Cbo.EndArray(); + CbObject Response = Cbo.Save(); + + return HttpReq.WriteResponse(HttpResponse::OK, Response); + } + break; + } + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "jobs", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + break; + + case HttpVerb::kPost: + { + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject RequestObject = LoadCompactBinaryObject(Payload); + + bool AllOk = true; + + std::vector<IoHash> NeedList; + + // TODO: auto-generate! + std::filesystem::path SandboxDir{"c:\\temp\\sandbox1"}; + zen::DeleteDirectories(SandboxDir); + zen::CreateDirectories(SandboxDir); + + for (auto Entry : RequestObject["files"sv]) + { + CbObjectView Ob = Entry.AsObjectView(); + + std::string_view FileName = Ob["file"sv].AsString(); + const IoHash FileHash = Ob["hash"sv].AsHash(); + uint64_t FileSize = Ob["size"sv].AsUInt64(); + + if (IoBuffer Chunk = m_CasStore.FindChunk(FileHash); !Chunk) + { + spdlog::debug("MISSING: {} {} {}", FileHash, FileName, FileSize); + AllOk = false; + + NeedList.push_back(FileHash); + } + else + { + std::filesystem::path FullPath = SandboxDir / FileName; + + const IoBuffer* Chunks[] = {&Chunk}; + + zen::WriteFile(FullPath, Chunks, 1); + } + } + + if (!AllOk) + { + // TODO: Could report all the missing pieces in the response here + return HttpReq.WriteResponse(HttpResponse::NotFound); + } + + std::wstring Executable = Utf8ToWide(RequestObject["cmd"].AsString()); + std::wstring Args = Utf8ToWide(RequestObject["args"].AsString()); + + std::filesystem::path ExeName = SandboxDir / Executable; + + BasicJob Job; + Job.SetWorkingDirectory(SandboxDir); + Job.SpawnJob(ExeName, Args); + Job.Wait(); + + return HttpReq.WriteResponse(HttpResponse::OK); + } + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost); +} + +HttpLaunchService::~HttpLaunchService() +{ +} + +const char* +HttpLaunchService::BaseUri() const +{ + return "/exec/"; +} + +void +HttpLaunchService::HandleRequest(HttpServerRequest& Request) +{ + if (m_Router.HandleRequest(Request) == false) + { + m_Log.warn("No route found for {0}", Request.RelativeUri()); + } +} + +} // namespace zen diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h new file mode 100644 index 000000000..5dd946eda --- /dev/null +++ b/zenserver/testing/launch.h @@ -0,0 +1,31 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/httpserver.h> + +#include <spdlog/spdlog.h> + +namespace zen { + +class CasStore; + +/** + * Process launcher for test executables + */ +class HttpLaunchService : public HttpService +{ +public: + HttpLaunchService(CasStore& Store); + ~HttpLaunchService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + +private: + spdlog::logger m_Log; + HttpRequestRouter m_Router; + CasStore& m_CasStore; +}; + +} // namespace zen diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp new file mode 100644 index 000000000..6b54f3d01 --- /dev/null +++ b/zenserver/upstream/jupiter.cpp @@ -0,0 +1,277 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "jupiter.h" + +#include <fmt/format.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/string.h> +#include <zencore/thread.h> + +// For some reason, these don't seem to stick, so we disable the warnings +//# define _SILENCE_CXX17_C_HEADER_DEPRECATION_WARNING 1 +//# define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS 1 +#pragma warning(push) +#pragma warning(disable : 4004) +#pragma warning(disable : 4996) +#include <cpr/cpr.h> +#pragma warning(pop) + +#if ZEN_PLATFORM_WINDOWS +# pragma comment(lib, "Crypt32.lib") +# pragma comment(lib, "Wldap32.lib") +#endif + +#include <spdlog/spdlog.h> +#include <json11.hpp> + +using namespace std::literals; +using namespace fmt::literals; + +namespace zen { + +namespace detail { + struct CloudCacheSessionState + { + CloudCacheSessionState(CloudCacheClient& Client) : OwnerClient(Client) {} + ~CloudCacheSessionState() {} + + void Reset() + { + std::string Auth; + OwnerClient.AcquireAccessToken(Auth); + + Session.SetBody({}); + Session.SetOption(cpr::Header{{"Authorization", Auth}}); + } + + CloudCacheClient& OwnerClient; + cpr::Session Session; + }; +} // namespace detail + +CloudCacheSession::CloudCacheSession(CloudCacheClient* OuterClient) : m_CacheClient(OuterClient) +{ + m_SessionState = m_CacheClient->AllocSessionState(); +} + +CloudCacheSession::~CloudCacheSession() +{ + m_CacheClient->FreeSessionState(m_SessionState); +} + +#define TESTING_PREFIX "aaaaa" + +IoBuffer +CloudCacheSession::Get(std::string_view BucketId, std::string_view Key) +{ + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl(); + Uri << "/api/v1/c/ddc/" << m_CacheClient->Namespace() << "/" << BucketId << "/" TESTING_PREFIX << Key << ".raw"; + + auto& Session = m_SessionState->Session; + Session.SetUrl(cpr::Url{Uri.c_str()}); + + cpr::Response Response = Session.Get(); + + if (!Response.error) + { + return IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()); + } + + return {}; +} + +void +CloudCacheSession::Put(std::string_view BucketId, std::string_view Key, IoBuffer Data) +{ + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl(); + Uri << "/api/v1/c/ddc/" << m_CacheClient->Namespace() << "/" << BucketId << "/" TESTING_PREFIX << Key; + + auto& Session = m_SessionState->Session; + + IoHash Hash = IoHash::HashMemory(Data.Data(), Data.Size()); + + std::string Auth; + m_CacheClient->AcquireAccessToken(Auth); + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption( + cpr::Header{{"Authorization", Auth}, {"X-Jupiter-IoHash", Hash.ToHexString()}, {"Content-Type", "application/octet-stream"}}); + Session.SetOption(cpr::Body{(const char*)Data.Data(), Data.Size()}); + + cpr::Response Response = Session.Put(); + + if (Response.error) + { + spdlog::warn("PUT failed: '{}'", Response.error.message); + } +} + +////////////////////////////////////////////////////////////////////////// + +std::string +CloudCacheAccessToken::GetAuthorizationHeaderValue() +{ + RwLock::SharedLockScope _(m_Lock); + + return "Bearer {}"_format(m_Token); +} + +inline void +CloudCacheAccessToken::SetToken(std::string_view Token) +{ + RwLock::ExclusiveLockScope _(m_Lock); + m_Token = Token; + ++m_Serial; +} + +////////////////////////////////////////////////////////////////////////// +// +// ServiceUrl: https://jupiter.devtools.epicgames.com +// Namespace: ue4.ddc +// OAuthClientId: 0oao91lrhqPiAlaGD0x7 +// OAuthProvider: https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token +// OAuthSecret: -GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d +// + +CloudCacheClient::CloudCacheClient(std::string_view ServiceUrl, + std::string_view Namespace, + std::string_view OAuthProvider, + std::string_view OAuthClientId, + std::string_view OAuthSecret) +: m_ServiceUrl(ServiceUrl) +, m_OAuthFullUri(OAuthProvider) +, m_Namespace(Namespace) +, m_DefaultBucket("default") +, m_OAuthClientId(OAuthClientId) +, m_OAuthSecret(OAuthSecret) +{ + if (!OAuthProvider.starts_with("http://"sv) && !OAuthProvider.starts_with("https://"sv)) + { + spdlog::warn("bad provider specification: '{}' - must be fully qualified"_format(OAuthProvider).c_str()); + m_IsValid = false; + + return; + } + + // Split into host and Uri substrings + + auto SchemePos = OAuthProvider.find("://"sv); + + if (SchemePos == std::string::npos) + { + spdlog::warn("Bad service URL passed to cloud cache client: '{}'", ServiceUrl); + m_IsValid = false; + + return; + } + + auto DomainEnd = OAuthProvider.find('/', /* also skip the :// */ SchemePos + 3); + + if (DomainEnd == std::string::npos) + { + spdlog::warn("Bad service URL passed to cloud cache client: '{}' no path delimiter found", ServiceUrl); + m_IsValid = false; + + return; + } + + m_OAuthDomain = OAuthProvider.substr(SchemePos + 3, DomainEnd - SchemePos - 3); // epicgames.okta.com + m_OAuthUriPath = OAuthProvider.substr(DomainEnd + 1); // oauth2/..../v1/token +} + +CloudCacheClient::~CloudCacheClient() +{ + RwLock::ExclusiveLockScope _(m_SessionStateLock); + + for (auto State : m_SessionStateCache) + { + delete State; + } +} + +bool +CloudCacheClient::AcquireAccessToken(std::string& AuthorizationHeaderValue) +{ + // TODO: check for expiration + + if (!m_IsValid) + { + ExtendableStringBuilder<128> OAuthFormData; + OAuthFormData << "client_id=" << m_OAuthClientId + << "&scope=cache_access&grant_type=client_credentials&client_secret=" << m_OAuthSecret; + + const uint32_t CurrentSerial = m_AccessToken.GetSerial(); + + static RwLock AuthMutex; + RwLock::ExclusiveLockScope _(AuthMutex); + + // Protect against redundant authentication operations + if (m_AccessToken.GetSerial() != CurrentSerial) + { + // TODO: this could verify that the token is actually valid and retry if not? + + return true; + } + + std::string data{OAuthFormData}; + + cpr::Response Response = + cpr::Post(cpr::Url{m_OAuthFullUri}, cpr::Header{{"Content-Type", "application/x-www-form-urlencoded"}}, cpr::Body{data}); + + std::string Body{std::move(Response.text)}; + + // Parse JSON response + + std::string JsonError; + json11::Json JsonResponse = json11::Json::parse(Body, /* out */ JsonError); + if (!JsonError.empty()) + { + spdlog::warn("failed to parse OAuth response: '{}'", JsonError); + + return false; + } + + std::string AccessToken = JsonResponse["access_token"].string_value(); + int ExpiryTimeSeconds = JsonResponse["expires_in"].int_value(); + + m_AccessToken.SetToken(AccessToken); + + m_IsValid = true; + } + + AuthorizationHeaderValue = m_AccessToken.GetAuthorizationHeaderValue(); + + return true; +} + +detail::CloudCacheSessionState* +CloudCacheClient::AllocSessionState() +{ + detail::CloudCacheSessionState* State = nullptr; + + if (RwLock::ExclusiveLockScope _(m_SessionStateLock); !m_SessionStateCache.empty()) + { + State = m_SessionStateCache.front(); + m_SessionStateCache.pop_front(); + } + + if (State == nullptr) + { + State = new detail::CloudCacheSessionState(*this); + } + + State->Reset(); + + return State; +} + +void +CloudCacheClient::FreeSessionState(detail::CloudCacheSessionState* State) +{ + RwLock::ExclusiveLockScope _(m_SessionStateLock); + m_SessionStateCache.push_front(State); +} + +} // namespace zen diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h new file mode 100644 index 000000000..dd01cfb86 --- /dev/null +++ b/zenserver/upstream/jupiter.h @@ -0,0 +1,97 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/refcount.h> +#include <zencore/thread.h> + +#include <atomic> +#include <list> +#include <memory> + +namespace zen { +namespace detail { + struct CloudCacheSessionState; +} + +class IoBuffer; +class CloudCacheClient; +struct IoHash; + +/** + * Cached access token, for use with `Authorization:` header + */ +struct CloudCacheAccessToken +{ + std::string GetAuthorizationHeaderValue(); + void SetToken(std::string_view Token); + + inline uint32_t GetSerial() const { return m_Serial.load(std::memory_order::memory_order_relaxed); } + +private: + RwLock m_Lock; + std::string m_Token; + std::atomic<uint32_t> m_Serial; +}; + +/** + * Context for performing Jupiter operations + * + * Maintains an HTTP connection so that subsequent operations don't need to go + * through the whole connection setup process + * + */ +class CloudCacheSession +{ +public: + CloudCacheSession(CloudCacheClient* OuterClient); + ~CloudCacheSession(); + + IoBuffer Get(std::string_view BucketId, std::string_view Key); + void Put(std::string_view BucketId, std::string_view Key, IoBuffer Data); + +private: + RefPtr<CloudCacheClient> m_CacheClient; + detail::CloudCacheSessionState* m_SessionState; +}; + +/** + * Jupiter upstream cache client + */ +class CloudCacheClient : public RefCounted +{ +public: + CloudCacheClient(std::string_view ServiceUrl, + std::string_view Namespace, + std::string_view OAuthProvider, + std::string_view OAuthClientId, + std::string_view OAuthSecret); + ~CloudCacheClient(); + + bool AcquireAccessToken(std::string& AuthorizationHeaderValue); + std::string_view Namespace() const { return m_Namespace; } + std::string_view DefaultBucket() const { return m_DefaultBucket; } + std::string_view ServiceUrl() const { return m_ServiceUrl; } + +private: + bool m_IsValid = false; + std::string m_ServiceUrl; + std::string m_OAuthDomain; + std::string m_OAuthUriPath; + std::string m_OAuthFullUri; + std::string m_Namespace; + std::string m_DefaultBucket; + std::string m_OAuthClientId; + std::string m_OAuthSecret; + CloudCacheAccessToken m_AccessToken; + + RwLock m_SessionStateLock; + std::list<detail::CloudCacheSessionState*> m_SessionStateCache; + + detail::CloudCacheSessionState* AllocSessionState(); + void FreeSessionState(detail::CloudCacheSessionState*); + + friend class CloudCacheSession; +}; + +} // namespace zen diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp new file mode 100644 index 000000000..7148715f2 --- /dev/null +++ b/zenserver/upstream/zen.cpp @@ -0,0 +1,291 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zen.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/fmtutils.h> +#include <zencore/stream.h> + +#include <spdlog/spdlog.h> +#include <xxhash.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +namespace detail { + struct MessageHeader + { + static const uint32_t kMagic = 0x11'99'77'22; + + uint32_t Magic = kMagic; + uint32_t Checksum = 0; + uint16_t MessageSize = 0; // Size *including* this field and the reserved field + uint16_t Reserved = 0; + + void SetPayload(const void* PayloadData, uint64_t PayloadSize) + { + memcpy(Payload(), PayloadData, PayloadSize); + MessageSize = gsl::narrow<uint16_t>(PayloadSize + sizeof MessageSize + sizeof Reserved); + Checksum = ComputeChecksum(); + } + + inline CbObject GetMessage() const + { + if (IsOk()) + { + MemoryView MessageView(Payload(), MessageSize - sizeof MessageSize - sizeof Reserved); + + CbValidateError ValidationResult = ValidateCompactBinary(MessageView, CbValidateMode::All); + + if (ValidationResult == CbValidateError::None) + { + return CbObject{SharedBuffer::MakeView(MessageView)}; + } + } + + return {}; + } + + uint32_t TotalSize() const { return MessageSize + sizeof Checksum + sizeof Magic; } + uint32_t ComputeChecksum() const { return gsl::narrow_cast<uint32_t>(XXH3_64bits(&MessageSize, MessageSize)); } + inline bool IsOk() const { return Magic == kMagic && Checksum == ComputeChecksum(); } + + private: + inline void* Payload() { return &Reserved + 1; } + inline const void* Payload() const { return &Reserved + 1; } + }; +} // namespace detail + +// Note that currently this just implements an UDP echo service for testing purposes + +Mesh::Mesh(asio::io_context& IoContext) : m_IoContext(IoContext) +{ +} + +Mesh::~Mesh() +{ + Stop(); +} + +void +Mesh::Start(uint16_t Port) +{ + ZEN_ASSERT(Port); + ZEN_ASSERT(m_Port == 0); + + m_Port = Port; + m_UdpSocket = std::make_unique<asio::ip::udp::socket>(m_IoContext, asio::ip::udp::endpoint(asio::ip::udp::v4(), m_Port)); + m_Thread = std::make_unique<std::thread>([this] { Run(); }); +}; + +void +Mesh::Stop() +{ + using namespace std::literals; + + if (!m_Port) + { + // Never started, nothing to do here + return; + } + + CbObjectWriter Msg; + Msg << "bye"sv << m_SessionId; + BroadcastPacket(Msg); + + m_State = kExiting; + + std::error_code Ec; + m_Timer.cancel(Ec); + + m_UdpSocket->close(Ec); + + m_IoContext.stop(); + + if (m_Thread) + { + m_Thread->join(); + m_Thread.reset(); + } +} + +void +Mesh::EnqueueTick() +{ + m_Timer.expires_after(std::chrono::seconds(10)); + + m_Timer.async_wait([&](const std::error_code& Ec) { + if (!Ec) + { + OnTick(); + } + else + { + if (m_State != kExiting) + { + spdlog::warn("Mesh timer error: {}", Ec.message()); + } + } + }); +} + +void +Mesh::OnTick() +{ + using namespace std::literals; + + CbObjectWriter Msg; + + // Basic service information + + Msg.BeginArray("s"); + Msg << m_SessionId << m_Port << /* event sequence # */ uint32_t(0); + Msg.EndArray(); + + BroadcastPacket(Msg); + + EnqueueTick(); +} + +void +Mesh::BroadcastPacket(CbObjectWriter& Obj) +{ + std::error_code ErrorCode; + + asio::ip::udp::socket BroadcastSocket(m_IoContext); + BroadcastSocket.open(asio::ip::udp::v4(), ErrorCode); + + if (!ErrorCode) + { + BroadcastSocket.set_option(asio::ip::udp::socket::reuse_address(true)); + BroadcastSocket.set_option(asio::socket_base::broadcast(true)); + + asio::ip::udp::endpoint BroadcastEndpoint(asio::ip::address_v4::broadcast(), m_Port); + + uint8_t MessageBuffer[kMaxMessageSize]; + detail::MessageHeader* Message = reinterpret_cast<detail::MessageHeader*>(MessageBuffer); + *Message = {}; + + MemoryOutStream MemOut; + BinaryWriter Writer(MemOut); + + Obj.Save(Writer); + + // TODO: check that it fits in a packet! + + Message->SetPayload(MemOut.Data(), MemOut.Size()); + + BroadcastSocket.send_to(asio::buffer(Message, Message->TotalSize()), BroadcastEndpoint); + BroadcastSocket.close(); + } + else + { + spdlog::warn("failed to open broadcast socket: {}", ErrorCode.message()); + } +} + +void +Mesh::Run() +{ + m_State = kRunning; + + EnqueueTick(); + + IssueReceive(); + m_IoContext.run(); +} + +void +Mesh::IssueReceive() +{ + using namespace std::literals; + + m_UdpSocket->async_receive_from( + asio::buffer(m_MessageBuffer, sizeof m_MessageBuffer), + m_SenderEndpoint, + [this](std::error_code ec, size_t BytesReceived) { + if (!ec && BytesReceived) + { + std::error_code ErrorCode; + std::string Sender = m_SenderEndpoint.address().to_string(ErrorCode); + + // Process message + + uint32_t& Magic = *reinterpret_cast<uint32_t*>(m_MessageBuffer); + + switch (Magic) + { + case detail::MessageHeader::kMagic: + { + detail::MessageHeader& Header = *reinterpret_cast<detail::MessageHeader*>(m_MessageBuffer); + + if (CbObject Msg = Header.GetMessage()) + { + const asio::ip::address& Ip = m_SenderEndpoint.address(); + + if (auto Field = Msg["s"sv]) + { + // Announce + + CbArrayView Ci = Field.AsArrayView(); + auto It = Ci.CreateViewIterator(); + + const Oid SessionId = It->AsObjectId(); + + if (SessionId != Oid::Zero && SessionId != m_SessionId) + { + const uint16_t Port = (++It)->AsUInt16(m_SenderEndpoint.port()); + const uint32_t Lsn = (++It)->AsUInt32(); + + spdlog::info("received hey from {} ({})", Sender, SessionId); + + RwLock::ExclusiveLockScope _(m_SessionsLock); + + PeerInfo& Info = m_KnownPeers[SessionId]; + + Info.LastSeen = std::time(nullptr); + Info.SessionId = SessionId; + + if (std::find(begin(Info.SeenOnIP), end(Info.SeenOnIP), Ip) == Info.SeenOnIP.end()) + { + Info.SeenOnIP.push_back(Ip); + } + } + } + else if (auto Bye = Msg["bye"sv]) + { + Oid SessionId = Field.AsObjectId(); + + spdlog::info("received bye from {} ({})", Sender, SessionId); + + // We could verify that it's sent from a known IP before erasing the + // session, if we want to be paranoid + + RwLock::ExclusiveLockScope _(m_SessionsLock); + + m_KnownPeers.erase(SessionId); + } + else + { + // Unknown message type, just ignore + } + } + else + { + spdlog::warn("received malformed message from {}", Sender); + } + } + break; + + default: + spdlog::warn("received malformed data from {}", Sender); + break; + } + + IssueReceive(); + } + }); +} + +} // namespace zen diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h new file mode 100644 index 000000000..75e29bf86 --- /dev/null +++ b/zenserver/upstream/zen.h @@ -0,0 +1,84 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/memory.h> +#include <zencore/thread.h> +#include <zencore/uid.h> +#include <zencore/zencore.h> + +#pragma warning(push) +#pragma warning(disable : 4127) +#include <tsl/robin_map.h> +#pragma warning(pop) + +#include <asio.hpp> + +#include <chrono> + +namespace zen { + +class CbObjectWriter; + +/** Zen mesh tracker + * + * Discovers and tracks local peers + */ +class Mesh +{ +public: + Mesh(asio::io_context& IoContext); + ~Mesh(); + + void Start(uint16_t Port); + void Stop(); + +private: + void Run(); + void IssueReceive(); + void EnqueueTick(); + void OnTick(); + void BroadcastPacket(CbObjectWriter&); + + enum State + { + kInitializing, + kRunning, + kExiting + }; + + static const int kMaxMessageSize = 2048; + static const int kMaxUpdateSize = 1400; // We'll try not to send messages larger than this + + std::atomic<State> m_State = kInitializing; + asio::io_context& m_IoContext; + std::unique_ptr<asio::ip::udp::socket> m_UdpSocket; + std::unique_ptr<asio::ip::udp::socket> m_BroadcastSocket; + asio::ip::udp::endpoint m_SenderEndpoint; + std::unique_ptr<std::thread> m_Thread; + uint16_t m_Port = 0; + uint8_t m_MessageBuffer[kMaxMessageSize]; + asio::high_resolution_timer m_Timer{m_IoContext}; + Oid m_SessionId{Oid::NewOid()}; + + struct PeerInfo + { + Oid SessionId; + std::time_t LastSeen; + std::vector<asio::ip::address> SeenOnIP; + }; + + RwLock m_SessionsLock; + tsl::robin_map<Oid, PeerInfo, Oid::Hasher> m_KnownPeers; +}; + +class ZenKvCacheClient +{ +public: + ZenKvCacheClient(); + ~ZenKvCacheClient(); + +private: +}; + +} // namespace zen diff --git a/zenserver/vfs.cpp b/zenserver/vfs.cpp new file mode 100644 index 000000000..71f0bbdda --- /dev/null +++ b/zenserver/vfs.cpp @@ -0,0 +1,898 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "vfs.h" + +#include <zencore/except.h> +#include <zencore/filesystem.h> +#include <zencore/snapshot_manifest.h> +#include <zencore/stream.h> +#include <zencore/windows.h> + +#include <map> + +#include <atlfile.h> +#include <projectedfslib.h> +#include <spdlog/spdlog.h> + +#pragma comment(lib, "projectedfslib.lib") + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +struct ProjFsCliOptions +{ + bool IsDebug = false; + bool IsClean = false; + std::string CasSpec; + std::string ManifestSpec; + std::string MountPoint; +}; + +struct GuidHasher +{ + size_t operator()(const GUID& Guid) const + { + static_assert(sizeof(GUID) == (sizeof(size_t) * 2)); + + const size_t* Ptr = reinterpret_cast<const size_t*>(&Guid); + + return Ptr[0] ^ Ptr[1]; + } +}; + +class ProjfsNamespace +{ +public: + HRESULT Initialize(const char* SnapshotSpec, const char* CasSpec) + { + std::filesystem::path ManifestSpec = zen::ManifestSpecToPath(SnapshotSpec); + + CAtlFile ManifestFile; + HRESULT hRes = ManifestFile.Create(ManifestSpec.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); + if (FAILED(hRes)) + { + spdlog::error("MANIFEST NOT FOUND!"); // TODO: add context + + return hRes; + } + + ULONGLONG FileLength = 0; + ManifestFile.GetSize(FileLength); + + std::vector<uint8_t> Data; + Data.resize(FileLength); + + ManifestFile.Read(Data.data(), (DWORD)Data.size()); + + zen::MemoryInStream MemoryStream(Data.data(), Data.size()); + + ReadManifest(/* out */ m_Manifest, MemoryStream); + + uint64_t TotalBytes = 0; + uint64_t TotalFiles = 0; + + m_Manifest.Root.VisitFiles([&](const zen::LeafNode& Node) { + TotalBytes += Node.FileSize; + TotalFiles++; + }); + + m_FileByteCount = TotalBytes; + m_FileCount = TotalFiles; + + // CAS root + + zen::CasStoreConfiguration Config; + Config.RootDirectory = CasSpec; + m_CasStore->Initialize(Config); + + return S_OK; + } + + struct LookupResult + { + const zen::TreeNode* TreeNode = nullptr; + const zen::LeafNode* LeafNode = nullptr; + }; + + bool IsOnCasDrive(const char* Path) + { + ZEN_UNUSED(Path); + + // TODO: programmatically determine of CAS and workspace path is on same drive! + return true; + } + + LookupResult LookupNode(const std::wstring& Name) const + { + if (Name.empty()) + return {nullptr}; + + zen::ExtendableWideStringBuilder<MAX_PATH> LocalName; + LocalName.Append(Name.c_str()); + + // Split components + + const wchar_t* PathComponents[MAX_PATH / 2]; + size_t PathComponentCount = 0; + + const size_t Length = Name.length(); + + wchar_t* Base = LocalName.Data(); + wchar_t* itStart = Base; + + for (int i = 0; i < Length; ++i) + { + if (Base[i] == '\\') + { + // Component separator + + Base[i] = L'\0'; + + PathComponents[PathComponentCount++] = itStart; + + itStart = Base + i + 1; + } + } + + // Push final component + if (Name.back() != L'\\') + PathComponents[PathComponentCount++] = itStart; + + const zen::TreeNode* Node = &m_Manifest.Root; + + if (PathComponentCount == 1) + { + if (PrjFileNameCompare(L"root", Name.c_str()) == 0) + return {Node}; + else + return {nullptr}; + } + + for (size_t i = 1; i < PathComponentCount; ++i) + { + const auto& part = PathComponents[i]; + + const zen::TreeNode* NextNode = nullptr; + + for (const zen::TreeNode& ChildNode : Node->Children) + { + if (PrjFileNameCompare(part, ChildNode.Name.c_str()) == 0) + { + NextNode = &ChildNode; + break; + } + } + + if (NextNode) + { + Node = NextNode; + + continue; + } + + if (i == PathComponentCount - 1) + { + for (const zen::LeafNode& Leaf : Node->Leaves) + { + if (PrjFileNameCompare(part, Leaf.Name.c_str()) == 0) + return {nullptr, &Leaf}; + } + } + + return {nullptr}; + } + + return {Node}; + } + + const zen::SnapshotManifest& Manifest() const { return m_Manifest; } + zen::CasStore& CasStore() { return *m_CasStore; } + + uint64_t FileCount() const { return m_FileCount; } + uint64_t FileByteCount() const { return m_FileByteCount; } + +private: + zen::SnapshotManifest m_Manifest; + std::unique_ptr<zen::CasStore> m_CasStore; + + size_t m_FileCount = 0; + size_t m_FileByteCount = 0; +}; + +/** Projected File System Provider + */ + +class ProjfsProvider +{ +public: + HRESULT ReadManifest(const char* ManifestSpec, const char* CasSpec); + HRESULT Initialize(std::filesystem::path RootPath, bool Clean); + void Cleanup(); + + struct Callbacks; + +private: + static void DebugPrint(const char* Format, ...); + + HRESULT StartDirEnum(const PRJ_CALLBACK_DATA* CallbackData, LPCGUID EnumerationId); + HRESULT EndDirEnum(const PRJ_CALLBACK_DATA* CallbackData, LPCGUID EnumerationId); + HRESULT GetDirEnum(const PRJ_CALLBACK_DATA* CallbackData, + LPCGUID EnumerationId, + LPCWSTR SearchExpression, + PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle); + HRESULT GetPlaceholderInformation(const PRJ_CALLBACK_DATA* CallbackData); + HRESULT GetFileStream(const PRJ_CALLBACK_DATA* CallbackData, UINT64 ByteOffset, UINT32 Length); + HRESULT QueryFileName(const PRJ_CALLBACK_DATA* CallbackData); + HRESULT NotifyOperation(const PRJ_CALLBACK_DATA* CallbackData, + BOOLEAN IsDirectory, + PRJ_NOTIFICATION NotificationType, + LPCWSTR DestinationFileName, + PRJ_NOTIFICATION_PARAMETERS* OperationParameters); + void CancelCommand(const PRJ_CALLBACK_DATA* CallbackData); + + class DirectoryEnumeration; + + zen::RwLock m_Lock; + std::unordered_map<GUID, std::unique_ptr<DirectoryEnumeration>, GuidHasher> m_DirectoryEnumerators; + ProjfsNamespace m_Namespace; + PRJ_NAMESPACE_VIRTUALIZATION_CONTEXT m_PrjContext = nullptr; + bool m_GenerateFullFiles = false; +}; + +class ProjfsProvider::DirectoryEnumeration +{ +public: + DirectoryEnumeration(ProjfsProvider* Outer, LPCGUID EnumerationGuid, const wchar_t* RelativePath) + : m_Outer(Outer) + , m_EnumerationId(*EnumerationGuid) + , m_Path(RelativePath) + { + ResetScan(); + } + + ~DirectoryEnumeration() {} + + void ResetScan() + { + // Restart enumeration from beginning + + m_InfoIterator = m_Infos.end(); + + const ProjfsNamespace::LookupResult Lookup = m_Outer->m_Namespace.LookupNode(m_Path); + + if (Lookup.TreeNode == nullptr && Lookup.LeafNode == nullptr) + return; + + if (Lookup.TreeNode) + { + const zen::TreeNode* RootNode = Lookup.TreeNode; + + // Populate info array + + FILETIME FileTime; + GetSystemTimeAsFileTime(&FileTime); + + for (const zen::TreeNode& ChildNode : RootNode->Children) + { + PRJ_FILE_BASIC_INFO Fbi{0}; + + Fbi.IsDirectory = TRUE; + Fbi.FileSize = 0; + Fbi.CreationTime = Fbi.LastAccessTime = Fbi.LastWriteTime = Fbi.ChangeTime = *((LARGE_INTEGER*)&FileTime); + Fbi.FileAttributes = FILE_ATTRIBUTE_DIRECTORY; + + m_Infos.insert({ChildNode.Name, Fbi}); + } + + for (const zen::LeafNode& Leaf : RootNode->Leaves) + { + PRJ_FILE_BASIC_INFO Fbi{0}; + + Fbi.IsDirectory = FALSE; + Fbi.FileSize = Leaf.FileSize; + Fbi.FileAttributes = FILE_ATTRIBUTE_NORMAL; + Fbi.CreationTime = Fbi.LastAccessTime = Fbi.LastWriteTime = Fbi.ChangeTime = + *reinterpret_cast<const LARGE_INTEGER*>(&Leaf.FileModifiedTime); + + m_Infos.insert({Leaf.Name, Fbi}); + } + } + + m_InfoIterator = m_Infos.begin(); + } + + HRESULT HandleRequest(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_opt_z_ LPCWSTR SearchExpression, + _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) + { + int EnumLimit = INT_MAX; + + DebugPrint("ENUM '%S' -> pattern %S\n", CallbackData->FilePathName, SearchExpression); + + HRESULT hRes = S_OK; + + if (CallbackData->Flags & PRJ_CB_DATA_FLAG_ENUM_RESTART_SCAN) + ResetScan(); + + if (m_InfoIterator == m_Infos.end()) + return S_OK; + + if (CallbackData->Flags & PRJ_CB_DATA_FLAG_ENUM_RETURN_SINGLE_ENTRY) + EnumLimit = 1; + + if (!m_Predicate) + { + if (SearchExpression) + { + bool isWild = PrjDoesNameContainWildCards(SearchExpression); + + if (isWild) + { + if (SearchExpression[0] == L'*' && SearchExpression[1] == L'\0') + { + // Trivial accept -- no need to change predicate from the default + } + else + { + m_SearchExpression = SearchExpression; + + m_Predicate = [this](LPCWSTR name) { return PrjFileNameMatch(name, m_SearchExpression.c_str()); }; + } + } + else + { + if (SearchExpression[0]) + { + // Look for specific name match (does this ever happen?) + + m_SearchExpression = SearchExpression; + + m_Predicate = [this](LPCWSTR name) { return PrjFileNameCompare(name, m_SearchExpression.c_str()) == 0; }; + } + } + } + } + + if (!m_Predicate) + m_Predicate = [](LPCWSTR) { return true; }; + + while (EnumLimit && m_InfoIterator != m_Infos.end()) + { + auto& ThisNode = *m_InfoIterator; + + auto& Name = ThisNode.first; + auto& Info = ThisNode.second; + + if (m_Predicate(Name.c_str())) + { + hRes = PrjFillDirEntryBuffer(Name.c_str(), &Info, DirEntryBufferHandle); + + if (hRes == HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER)) + return S_OK; + + if (FAILED(hRes)) + break; + + --EnumLimit; + } + + ++m_InfoIterator; + } + + return hRes; + } + +private: + ProjfsProvider* m_Outer = nullptr; + const std::wstring m_Path; + const GUID m_EnumerationId; + + // We need to maintain an ordered list of directory items since the + // ProjFS enumeration code gets confused otherwise and ends up producing + // multiple entries for the same file if there's a 'hydrated' version + // present. + + struct FilenameLess + { + bool operator()(const std::wstring& Lhs, const std::wstring& Rhs) const { return PrjFileNameCompare(Lhs.c_str(), Rhs.c_str()) < 0; } + }; + + typedef std::map<std::wstring, PRJ_FILE_BASIC_INFO, FilenameLess> FileInfoMap_t; + + FileInfoMap_t m_Infos; + FileInfoMap_t::iterator m_InfoIterator; + + std::wstring m_SearchExpression; + std::function<bool(LPCWSTR name)> m_Predicate; +}; + +////////////////////////////////////////////////////////////////////////// +// Callback forwarding functions +// + +struct ProjfsProvider::Callbacks +{ + static HRESULT CALLBACK StartDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ const GUID* EnumerationId) + { + return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->StartDirEnum(CallbackData, EnumerationId); + } + + static HRESULT CALLBACK EndDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ LPCGUID EnumerationId) + { + return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->EndDirEnum(CallbackData, EnumerationId); + } + + static HRESULT CALLBACK GetDirEnum(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ LPCGUID EnumerationId, + _In_opt_z_ LPCWSTR SearchExpression, + _In_ PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) + { + return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext) + ->GetDirEnum(CallbackData, EnumerationId, SearchExpression, DirEntryBufferHandle); + } + + static HRESULT CALLBACK GetPlaceholderInformation(_In_ const PRJ_CALLBACK_DATA* CallbackData) + { + return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->GetPlaceholderInformation(CallbackData); + } + + static HRESULT CALLBACK GetFileStream(_In_ const PRJ_CALLBACK_DATA* CallbackData, _In_ UINT64 ByteOffset, _In_ UINT32 Length) + { + return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->GetFileStream(CallbackData, ByteOffset, Length); + } + + static HRESULT CALLBACK QueryFileName(_In_ const PRJ_CALLBACK_DATA* CallbackData) + { + return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->QueryFileName(CallbackData); + } + + static HRESULT CALLBACK NotifyOperation(_In_ const PRJ_CALLBACK_DATA* CallbackData, + _In_ BOOLEAN IsDirectory, + _In_ PRJ_NOTIFICATION NotificationType, + _In_opt_ LPCWSTR DestinationFileName, + _Inout_ PRJ_NOTIFICATION_PARAMETERS* OperationParameters) + { + return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext) + ->NotifyOperation(CallbackData, IsDirectory, NotificationType, DestinationFileName, OperationParameters); + } + + static VOID CALLBACK CancelCommand(_In_ const PRJ_CALLBACK_DATA* CallbackData) + { + return reinterpret_cast<ProjfsProvider*>(CallbackData->InstanceContext)->CancelCommand(CallbackData); + } +}; + +// {6EEB94E4-3EF3-4C1C-AF15-D7FF64C19A4F} +static const GUID ProviderGuid = {0x6eeb94e4, 0x3ef3, 0x4c1c, {0xaf, 0x15, 0xd7, 0xff, 0x64, 0xc1, 0x9a, 0x4f}}; + +void +ProjfsProvider::DebugPrint(const char* FmtString, ...) +{ + va_list vl; + va_start(vl, FmtString); + +#if 0 + vprintf(FmtString, vl); +#endif + + va_end(vl); +} + +HRESULT +ProjfsProvider::Initialize(std::filesystem::path RootPath, bool Clean) +{ + PRJ_PLACEHOLDER_VERSION_INFO Pvi = {}; + Pvi.ContentID[0] = 1; + + if (Clean && std::filesystem::exists(RootPath)) + { + printf("Cleaning '%S'...", RootPath.c_str()); + + bool success = zen::DeleteDirectories(RootPath); + + if (!success) + { + printf(" retrying..."); + + success = zen::DeleteDirectories(RootPath); + + // Failed? + } + + printf(" done!\n"); + } + + bool RootDirectoryCreated = false; + +retry: + if (!std::filesystem::exists(RootPath)) + { + zen::CreateDirectories(RootPath); + } + + { + HRESULT hRes = PrjMarkDirectoryAsPlaceholder(RootPath.c_str(), nullptr, &Pvi, &ProviderGuid); + + if (FAILED(hRes)) + { + if (hRes == HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND) && !RootDirectoryCreated) + { + printf("Creating '%S'...", RootPath.c_str()); + + std::filesystem::create_directories(RootPath.c_str()); + + RootDirectoryCreated = true; + + printf("done!\n"); + + goto retry; + } + else if (hRes == HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND)) + { + throw zen::WindowsException(hRes, "Failed to initialize root placeholder"); + } + + // Ignore error, problems will be reported below anyway + } + } + + // Callbacks + + PRJ_CALLBACKS cbs = {}; + + cbs.StartDirectoryEnumerationCallback = Callbacks::StartDirEnum; + cbs.EndDirectoryEnumerationCallback = Callbacks::EndDirEnum; + cbs.GetDirectoryEnumerationCallback = Callbacks::GetDirEnum; + cbs.GetPlaceholderInfoCallback = Callbacks::GetPlaceholderInformation; + cbs.GetFileDataCallback = Callbacks::GetFileStream; + cbs.QueryFileNameCallback = Callbacks::QueryFileName; + cbs.NotificationCallback = Callbacks::NotifyOperation; + cbs.CancelCommandCallback = Callbacks::CancelCommand; + + // Parameters + + const PRJ_NOTIFY_TYPES dwNotifications = PRJ_NOTIFY_FILE_OPENED | PRJ_NOTIFY_NEW_FILE_CREATED | PRJ_NOTIFY_FILE_OVERWRITTEN | + PRJ_NOTIFY_PRE_DELETE | PRJ_NOTIFY_PRE_RENAME | PRJ_NOTIFY_PRE_SET_HARDLINK | + PRJ_NOTIFY_FILE_RENAMED | PRJ_NOTIFY_HARDLINK_CREATED | + PRJ_NOTIFY_FILE_HANDLE_CLOSED_NO_MODIFICATION | PRJ_NOTIFY_FILE_HANDLE_CLOSED_FILE_MODIFIED | + PRJ_NOTIFY_FILE_HANDLE_CLOSED_FILE_DELETED | PRJ_NOTIFY_FILE_PRE_CONVERT_TO_FULL; + + PRJ_NOTIFICATION_MAPPING Mappings[] = {{dwNotifications, L"root"}}; + + PRJ_STARTVIRTUALIZING_OPTIONS SvOptions = {}; + + SvOptions.Flags = PRJ_FLAG_NONE; + SvOptions.PoolThreadCount = 8; + SvOptions.ConcurrentThreadCount = 8; + SvOptions.NotificationMappings = Mappings; + SvOptions.NotificationMappingsCount = 1; + + HRESULT hRes = PrjStartVirtualizing(RootPath.c_str(), &cbs, this, &SvOptions, &m_PrjContext); + + if (SUCCEEDED(hRes)) + { + // Create dummy 'root' directory for now until I figure out how to + // invalidate entire trees (ProjFS won't allow invalidation of the + // entire provider tree). + + PRJ_PLACEHOLDER_INFO pli{}; + pli.FileBasicInfo.IsDirectory = TRUE; + pli.FileBasicInfo.FileAttributes = FILE_ATTRIBUTE_DIRECTORY; + pli.VersionInfo = Pvi; + + hRes = PrjWritePlaceholderInfo(m_PrjContext, L"root", &pli, sizeof pli); + } + + if (SUCCEEDED(hRes)) + { + spdlog::info("Successfully mounted snapshot at '{}'!", WideToUtf8(RootPath.c_str())); + } + else + { + spdlog::info("Failed mounting snapshot at '{}'!", WideToUtf8(RootPath.c_str())); + } + + return hRes; +} + +void +ProjfsProvider::Cleanup() +{ + PrjStopVirtualizing(m_PrjContext); +} + +HRESULT +ProjfsProvider::ReadManifest(const char* ManifestSpec, const char* CasSpec) +{ + printf("Initializing from manifest '%s'\n", ManifestSpec); + + m_Namespace.Initialize(ManifestSpec, CasSpec); + + return S_OK; +} + +HRESULT +ProjfsProvider::StartDirEnum(const PRJ_CALLBACK_DATA* CallbackData, LPCGUID EnumerationId) +{ + zen::RwLock::ExclusiveLockScope _(m_Lock); + + m_DirectoryEnumerators[*EnumerationId] = std::make_unique<DirectoryEnumeration>(this, EnumerationId, CallbackData->FilePathName); + + return S_OK; +} + +HRESULT +ProjfsProvider::EndDirEnum(const PRJ_CALLBACK_DATA* CallbackData, LPCGUID EnumerationId) +{ + ZEN_UNUSED(CallbackData); + ZEN_UNUSED(EnumerationId); + + zen::RwLock::ExclusiveLockScope _(m_Lock); + + m_DirectoryEnumerators.erase(*EnumerationId); + + return S_OK; +} + +HRESULT +ProjfsProvider::GetDirEnum(const PRJ_CALLBACK_DATA* CallbackData, + LPCGUID EnumerationId, + LPCWSTR SearchExpression, + PRJ_DIR_ENTRY_BUFFER_HANDLE DirEntryBufferHandle) +{ + DirectoryEnumeration* directoryEnumerator; + + { + zen::RwLock::SharedLockScope _(m_Lock); + + auto it = m_DirectoryEnumerators.find(*EnumerationId); + + if (it == m_DirectoryEnumerators.end()) + return E_FAIL; // No enumerator associated with specified GUID + + directoryEnumerator = (*it).second.get(); + } + + return directoryEnumerator->HandleRequest(CallbackData, SearchExpression, DirEntryBufferHandle); +} + +HRESULT +ProjfsProvider::GetPlaceholderInformation(const PRJ_CALLBACK_DATA* CallbackData) +{ + ProjfsNamespace::LookupResult result = m_Namespace.LookupNode(CallbackData->FilePathName); + + if (auto Leaf = result.LeafNode) + { + PRJ_PLACEHOLDER_INFO PlaceholderInfo = {}; + + LARGE_INTEGER FileTime; + FileTime.QuadPart = Leaf->FileModifiedTime; + + PlaceholderInfo.FileBasicInfo.ChangeTime = FileTime; + PlaceholderInfo.FileBasicInfo.CreationTime = FileTime; + PlaceholderInfo.FileBasicInfo.LastAccessTime = FileTime; + PlaceholderInfo.FileBasicInfo.LastWriteTime = FileTime; + PlaceholderInfo.FileBasicInfo.FileSize = Leaf->FileSize; + PlaceholderInfo.FileBasicInfo.IsDirectory = 0; + PlaceholderInfo.FileBasicInfo.FileAttributes = FILE_ATTRIBUTE_NORMAL; + + HRESULT hRes = PrjWritePlaceholderInfo(m_PrjContext, CallbackData->FilePathName, &PlaceholderInfo, sizeof PlaceholderInfo); + + return hRes; + } + + if (auto node = result.TreeNode) + { + PRJ_PLACEHOLDER_INFO PlaceholderInfo = {}; + + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + + LARGE_INTEGER FileTime; + FileTime.QuadPart = UINT64(ft.dwHighDateTime) << 32 | ft.dwLowDateTime; + + PlaceholderInfo.FileBasicInfo.ChangeTime = FileTime; + PlaceholderInfo.FileBasicInfo.CreationTime = FileTime; + PlaceholderInfo.FileBasicInfo.LastAccessTime = FileTime; + PlaceholderInfo.FileBasicInfo.LastWriteTime = FileTime; + PlaceholderInfo.FileBasicInfo.IsDirectory = TRUE; + PlaceholderInfo.FileBasicInfo.FileAttributes = FILE_ATTRIBUTE_DIRECTORY; + + HRESULT hRes = PrjWritePlaceholderInfo(m_PrjContext, CallbackData->FilePathName, &PlaceholderInfo, sizeof PlaceholderInfo); + + return hRes; + } + + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); +} + +HRESULT +ProjfsProvider::GetFileStream(const PRJ_CALLBACK_DATA* CallbackData, UINT64 ByteOffset, UINT32 Length) +{ + ProjfsNamespace::LookupResult result = m_Namespace.LookupNode(CallbackData->FilePathName); + + if (const zen::LeafNode* leaf = result.LeafNode) + { + zen::CasStore& casStore = m_Namespace.CasStore(); + + const zen::IoHash& ChunkHash = leaf->ChunkHash; + + zen::IoBuffer Chunk = casStore.FindChunk(ChunkHash); + + if (!Chunk) + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); + + if (m_GenerateFullFiles) + { + DWORD chunkSize = (DWORD)Chunk.Size(); + + zen::StringBuilder<66> b3string; + DebugPrint("GET FILE STREAM: %s -> %d '%S'\n", ChunkHash.ToHexString(b3string).c_str(), chunkSize, CallbackData->FilePathName); + + // TODO: implement support for chunks > 4GB + ZEN_ASSERT(chunkSize == Chunk.Size()); + + HRESULT hRes = PrjWriteFileData(m_PrjContext, &CallbackData->DataStreamId, (PVOID)Chunk.Data(), 0, chunkSize); + + return hRes; + } + else + { + HRESULT hRes = PrjWriteFileData(m_PrjContext, + &CallbackData->DataStreamId, + (PVOID)(reinterpret_cast<const uint8_t*>(Chunk.Data()) + ByteOffset), + ByteOffset, + Length); + + return hRes; + } + } + + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); +} + +HRESULT +ProjfsProvider::QueryFileName(const PRJ_CALLBACK_DATA* CallbackData) +{ + ProjfsNamespace::LookupResult result = m_Namespace.LookupNode(CallbackData->FilePathName); + + if (result.LeafNode || result.TreeNode) + return S_OK; + + return HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND); +} + +HRESULT +ProjfsProvider::NotifyOperation(const PRJ_CALLBACK_DATA* CallbackData, + BOOLEAN IsDirectory, + PRJ_NOTIFICATION NotificationType, + LPCWSTR DestinationFileName, + PRJ_NOTIFICATION_PARAMETERS* OperationParameters) +{ + ZEN_UNUSED(DestinationFileName); + + switch (NotificationType) + { + case PRJ_NOTIFICATION_FILE_OPENED: + { + auto& pc = OperationParameters->PostCreate; + + DebugPrint("*** OPEN: %s %08x '%S'\n", IsDirectory ? "(DIR)" : "-FILE", pc.NotificationMask, CallbackData->FilePathName); + } + break; + + case PRJ_NOTIFICATION_NEW_FILE_CREATED: + { + auto& pc = OperationParameters->PostCreate; + + DebugPrint("*** NEW : %s %08x '%S'\n", IsDirectory ? "(DIR)" : "-FILE", pc.NotificationMask, CallbackData->FilePathName); + } + break; + + case PRJ_NOTIFICATION_FILE_OVERWRITTEN: + { + auto& pc = OperationParameters->PostCreate; + + DebugPrint("*** OVER: %s %08x '%S'\n", IsDirectory ? "(DIR)" : "-FILE", pc.NotificationMask, CallbackData->FilePathName); + } + break; + + case PRJ_NOTIFICATION_PRE_DELETE: + { + if (wcsstr(CallbackData->FilePathName, L"en-us")) + DebugPrint("*** PRE DELETE '%S'\n", CallbackData->FilePathName); + + DebugPrint("*** PRE DELETE '%S'\n", CallbackData->FilePathName); + } + break; + + case PRJ_NOTIFICATION_PRE_RENAME: + DebugPrint("*** PRE RENAME '%S'\n", CallbackData->FilePathName); + break; + + case PRJ_NOTIFICATION_PRE_SET_HARDLINK: + DebugPrint("*** PRE SET HARDLINK '%S'\n", CallbackData->FilePathName); + break; + + case PRJ_NOTIFICATION_FILE_RENAMED: + DebugPrint("*** FILE RENAMED '%S'\n", CallbackData->FilePathName); + break; + + case PRJ_NOTIFICATION_HARDLINK_CREATED: + DebugPrint("*** HARDLINK RENAMED '%S'\n", CallbackData->FilePathName); + break; + + case PRJ_NOTIFICATION_FILE_HANDLE_CLOSED_NO_MODIFICATION: + DebugPrint("*** FILE CLOSED NO CHANGE '%S'\n", CallbackData->FilePathName); + break; + + case PRJ_NOTIFICATION_FILE_HANDLE_CLOSED_FILE_MODIFIED: + { + // const auto& handleClose = OperationParameters->FileDeletedOnHandleClose; + + DebugPrint("*** FILE CLOSED MODIFIED '%S'\n", CallbackData->FilePathName); + } + break; + + case PRJ_NOTIFICATION_FILE_HANDLE_CLOSED_FILE_DELETED: + { + // const auto& handleClose = OperationParameters->FileDeletedOnHandleClose; + + DebugPrint("*** FILE CLOSED DELETED '%S'\n", CallbackData->FilePathName); + } + break; + + case PRJ_NOTIFICATION_FILE_PRE_CONVERT_TO_FULL: + DebugPrint("*** FILE PRE CONVERT FULL '%S'\n", CallbackData->FilePathName); + break; + } + + return S_OK; +} + +void +ProjfsProvider::CancelCommand(const PRJ_CALLBACK_DATA* CallbackData) +{ + ZEN_UNUSED(CallbackData); +} + +////////////////////////////////////////////////////////////////////////// + +struct Vfs::VfsImpl +{ + void Initialize() { m_PrjProvider.Initialize("E:\\VFS_Test", /* clean */ true); } + void Start() {} + void Stop() {} + +private: + ProjfsProvider m_PrjProvider; +}; + +////////////////////////////////////////////////////////////////////////// + +Vfs::Vfs() : m_Impl(new VfsImpl) +{ +} + +Vfs::~Vfs() +{ +} + +void +Vfs::Initialize() +{ + m_Impl->Initialize(); +} + +void +Vfs::Start() +{ +} + +void +Vfs::Stop() +{ +} + +} // namespace zen diff --git a/zenserver/vfs.h b/zenserver/vfs.h new file mode 100644 index 000000000..e77ff381b --- /dev/null +++ b/zenserver/vfs.h @@ -0,0 +1,31 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/httpserver.h> +#include <zenstore/CAS.h> + +namespace zen { + +/** + * Virtual File System serving + */ + +class Vfs +{ +public: + Vfs(); + ~Vfs(); + + void Initialize(); + + void Start(); + void Stop(); + +private: + struct VfsImpl; + + std::unique_ptr<VfsImpl> m_Impl; +}; + +} // namespace zen diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp new file mode 100644 index 000000000..934fd95bc --- /dev/null +++ b/zenserver/zenserver.cpp @@ -0,0 +1,278 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/httpserver.h> +#include <zencore/iobuffer.h> +#include <zencore/refcount.h> +#include <zencore/string.h> +#include <zencore/thread.h> +#include <zencore/timer.h> +#include <zencore/windows.h> +#include <zenstore/cas.h> + +#include <fmt/format.h> +#include <mimalloc-new-delete.h> +#include <mimalloc.h> +#include <spdlog/spdlog.h> +#include <asio.hpp> +#include <list> +#include <lua.hpp> +#include <optional> +#include <regex> +#include <unordered_map> + +////////////////////////////////////////////////////////////////////////// +// We don't have any doctest code in this file but this is needed to bring +// in some shared code into the executable + +#define DOCTEST_CONFIG_IMPLEMENT +#include <doctest/doctest.h> +#undef DOCTEST_CONFIG_IMPLEMENT + +////////////////////////////////////////////////////////////////////////// + +#include "casstore.h" +#include "config.h" +#include "diag/crashreport.h" +#include "diag/logging.h" + +////////////////////////////////////////////////////////////////////////// +// Services +// + +#include "admin/admin.h" +#include "cache/kvcache.h" +#include "cache/structuredcache.h" +#include "diag/diagsvcs.h" +#include "experimental/usnjournal.h" +#include "projectstore.h" +#include "testing/launch.h" +#include "upstream/jupiter.h" +#include "upstream/zen.h" +#include "zenstore/gc.h" +#include "zenstore/scrub.h" + +#define ZEN_APP_NAME "Zen store" + +class ZenServer +{ +public: + void Initialize(int BasePort, int ParentPid) + { + using namespace fmt::literals; + spdlog::info(ZEN_APP_NAME " initializing"); + + if (ParentPid) + { + m_Process.Initialize(ParentPid); + } + + // Prototype config system, let's see how this pans out + + ZenServiceConfig ServiceConfig; + ParseServiceConfig(m_DataRoot, /* out */ ServiceConfig); + + // Ok so now we're configured, let's kick things off + + zen::CasStoreConfiguration Config; + Config.RootDirectory = m_DataRoot / "CAS"; + + m_CasStore->Initialize(Config); + + spdlog::info("instantiating project service"); + + m_ProjectStore = new zen::ProjectStore(*m_CasStore, m_DataRoot / "Builds"); + m_HttpProjectService.reset(new zen::HttpProjectService{*m_CasStore, m_ProjectStore}); + m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore); + + m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore); + + if (ServiceConfig.LegacyCacheEnabled) + { + spdlog::info("instantiating legacy cache service"); + m_CacheService.reset(new zen::HttpKvCacheService()); + } + else + { + spdlog::info("NOT instantiating legacy cache service"); + } + + if (ServiceConfig.StructuredCacheEnabled) + { + spdlog::info("instantiating structured cache service"); + m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(m_DataRoot / "cache", *m_CasStore)); + } + else + { + spdlog::info("NOT instantiating structured cache service"); + } + + m_Http.Initialize(BasePort); + m_Http.AddEndpoint(m_HealthService); + m_Http.AddEndpoint(m_TestService); + m_Http.AddEndpoint(m_AdminService); + + if (m_HttpProjectService) + { + m_Http.AddEndpoint(*m_HttpProjectService); + } + + m_Http.AddEndpoint(m_CasService); + + if (m_CacheService) + { + spdlog::info("instantiating legacy cache service"); + m_Http.AddEndpoint(*m_CacheService); + } + + if (m_StructuredCacheService) + { + m_Http.AddEndpoint(*m_StructuredCacheService); + } + + if (m_HttpLaunchService) + { + m_Http.AddEndpoint(*m_HttpLaunchService); + } + + // Experimental + // + // m_ZenMesh.Start(1337); + } + + void Run() + { + if (m_Process.IsValid()) + { + EnqueueTimer(); + } + + if (!m_TestMode) + { + spdlog::info("__________ _________ __ "); + spdlog::info("\\____ /____ ____ / _____// |_ ___________ ____ "); + spdlog::info(" / // __ \\ / \\ \\_____ \\\\ __\\/ _ \\_ __ \\_/ __ \\ "); + spdlog::info(" / /\\ ___/| | \\ / \\| | ( <_> ) | \\/\\ ___/ "); + spdlog::info("/_______ \\___ >___| / /_______ /|__| \\____/|__| \\___ >"); + spdlog::info(" \\/ \\/ \\/ \\/ \\/ "); + } + + spdlog::info(ZEN_APP_NAME " now running"); + + m_Http.Run(m_TestMode); + + spdlog::info(ZEN_APP_NAME " exiting"); + + m_IoContext.stop(); + } + + void RequestExit(int ExitCode) + { + RequestApplicationExit(ExitCode); + m_Http.RequestExit(); + } + + void Cleanup() { spdlog::info(ZEN_APP_NAME " cleaning up"); } + + void SetTestMode(bool State) { m_TestMode = State; } + void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } + + void EnqueueTimer() + { + m_PidCheckTimer.expires_after(std::chrono::seconds(1)); + m_PidCheckTimer.async_wait([this](const asio::error_code&) { CheckOwnerPid(); }); + } + + void CheckOwnerPid() + { + if (m_Process.IsRunning()) + { + EnqueueTimer(); + } + else + { + spdlog::info(ZEN_APP_NAME " exiting since parent process id {} is gone", m_Process.Pid()); + + RequestExit(0); + } + } + +private: + bool m_TestMode = false; + std::filesystem::path m_DataRoot; + asio::io_context m_IoContext; + asio::steady_timer m_PidCheckTimer{m_IoContext}; + zen::Process m_Process; + + zen::HttpServer m_Http; + std::unique_ptr<zen::CasStore> m_CasStore{zen::CreateCasStore()}; + zen::CasGc m_Gc{*m_CasStore}; + zen::CasScrubber m_Scrubber{*m_CasStore}; + HttpTestService m_TestService; + zen::HttpCasService m_CasService{*m_CasStore}; + std::unique_ptr<zen::HttpKvCacheService> m_CacheService; + zen::RefPtr<zen::ProjectStore> m_ProjectStore; + zen::Ref<zen::LocalProjectService> m_LocalProjectService; + std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService; + std::unique_ptr<zen::HttpProjectService> m_HttpProjectService; + std::unique_ptr<zen::HttpStructuredCacheService> m_StructuredCacheService; + HttpAdminService m_AdminService; + HttpHealthService m_HealthService; + zen::Mesh m_ZenMesh{m_IoContext}; +}; + +int +main(int argc, char* argv[]) +{ + mi_version(); + + ZenServerOptions GlobalOptions; + ParseGlobalCliOptions(argc, argv, GlobalOptions); + InitializeCrashReporting(GlobalOptions.DataDir / "crashdumps"); + InitializeLogging(GlobalOptions); + + spdlog::info("zen cache server starting on port {}", GlobalOptions.BasePort); + + try + { + std::unique_ptr<std::thread> ShutdownThread; + std::unique_ptr<zen::NamedEvent> ShutdownEvent; + + ZenServer Cache; + Cache.SetDataRoot(GlobalOptions.DataDir); + Cache.SetTestMode(GlobalOptions.IsTest); + Cache.Initialize(GlobalOptions.BasePort, GlobalOptions.OwnerPid); + + if (!GlobalOptions.ChildId.empty()) + { + zen::ExtendableStringBuilder<64> ShutdownEventName; + ShutdownEventName << GlobalOptions.ChildId << "_Shutdown"; + ShutdownEvent.reset(new zen::NamedEvent{ShutdownEventName}); + + zen::NamedEvent ParentEvent{GlobalOptions.ChildId}; + ParentEvent.Set(); + + ShutdownThread.reset(new std::thread{[&] { + ShutdownEvent->Wait(); + spdlog::info("shutdown signal received"); + Cache.RequestExit(0); + }}); + } + + Cache.Run(); + Cache.Cleanup(); + + if (ShutdownEvent) + { + ShutdownEvent->Set(); + ShutdownThread->join(); + } + } + catch (std::exception& e) + { + SPDLOG_CRITICAL("Caught exception in main: {}", e.what()); + } + + return 0; +} diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj new file mode 100644 index 000000000..b47ec2f04 --- /dev/null +++ b/zenserver/zenserver.vcxproj @@ -0,0 +1,150 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <VCProjectVersion>15.0</VCProjectVersion> + <ProjectGuid>{8398D81C-B1B6-4327-82B1-06EACB8A144F}</ProjectGuid> + <Keyword>Win32Proj</Keyword> + <RootNamespace>zenserver</RootNamespace> + <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>true</UseDebugLibraries> + <PlatformToolset>v142</PlatformToolset> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>false</UseDebugLibraries> + <PlatformToolset>v142</PlatformToolset> + <WholeProgramOptimization>true</WholeProgramOptimization> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="Shared"> + </ImportGroup> + <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="..\zenfs_common.props" /> + <Import Project="..\zen_base_debug.props" /> + </ImportGroup> + <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="..\zenfs_common.props" /> + <Import Project="..\zen_base_release.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <LinkIncremental>false</LinkIncremental> + <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);..\3rdparty\TraceLog\Public</IncludePath> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <LinkIncremental>true</LinkIncremental> + <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);..\3rdparty\TraceLog\Public</IncludePath> + </PropertyGroup> + <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <VcpkgEnableManifest>true</VcpkgEnableManifest> + <VcpkgUseStatic>true</VcpkgUseStatic> + </PropertyGroup> + <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <VcpkgEnableManifest>true</VcpkgEnableManifest> + <VcpkgUseStatic>true</VcpkgUseStatic> + </PropertyGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <Optimization>MaxSpeed</Optimization> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <ConformanceMode>true</ConformanceMode> + <AdditionalIncludeDirectories>..\zencore\include;..\zenstore\include;.</AdditionalIncludeDirectories> + <LanguageStandard>stdcpplatest</LanguageStandard> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + <GenerateDebugInformation>true</GenerateDebugInformation> + <UACExecutionLevel>RequireAdministrator</UACExecutionLevel> + </Link> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <Optimization>Disabled</Optimization> + <SDLCheck>true</SDLCheck> + <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <ConformanceMode>true</ConformanceMode> + <AdditionalIncludeDirectories>..\zencore\include;..\zenstore\include;.</AdditionalIncludeDirectories> + <LanguageStandard>stdcpplatest</LanguageStandard> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation>true</GenerateDebugInformation> + <UACExecutionLevel>RequireAdministrator</UACExecutionLevel> + <DelayLoadDLLs>projectedfslib.dll</DelayLoadDLLs> + </Link> + </ItemDefinitionGroup> + <ItemGroup> + <ClInclude Include="admin\admin.h" /> + <ClInclude Include="cache\structuredcache.h" /> + <ClInclude Include="config.h" /> + <ClInclude Include="diag\crashreport.h" /> + <ClInclude Include="diag\logging.h" /> + <ClInclude Include="upstream\jupiter.h" /> + <ClInclude Include="projectstore.h" /> + <ClInclude Include="cache\cacheagent.h" /> + <ClInclude Include="cache\cachestore.h" /> + <ClInclude Include="cache\kvcache.h" /> + <ClInclude Include="testing\launch.h" /> + <ClInclude Include="casstore.h" /> + <ClInclude Include="diag\diagsvcs.h" /> + <ClInclude Include="experimental\usnjournal.h" /> + <ClInclude Include="targetver.h" /> + <ClInclude Include="upstream\zen.h" /> + <ClInclude Include="vfs.h" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="cache\kvcache.cpp" /> + <ClCompile Include="cache\structuredcache.cpp" /> + <ClCompile Include="config.cpp" /> + <ClCompile Include="diag\crashreport.cpp" /> + <ClCompile Include="diag\logging.cpp" /> + <ClCompile Include="projectstore.cpp" /> + <ClCompile Include="cache\cacheagent.cpp" /> + <ClCompile Include="upstream\jupiter.cpp" /> + <ClCompile Include="testing\launch.cpp" /> + <ClCompile Include="cache\cachestore.cpp" /> + <ClCompile Include="casstore.cpp" /> + <ClCompile Include="experimental\usnjournal.cpp" /> + <ClCompile Include="upstream\zen.cpp" /> + <ClCompile Include="vfs.cpp" /> + <ClCompile Include="zenserver.cpp" /> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="..\zencore\zencore.vcxproj"> + <Project>{d75bf9ab-c61e-4fff-ad59-1563430f05e2}</Project> + </ProjectReference> + <ProjectReference Include="..\zenstore\zenstore.vcxproj"> + <Project>{26cbbaeb-14c1-4efc-877d-80f48215651c}</Project> + </ProjectReference> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + </ImportGroup> +</Project>
\ No newline at end of file diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters new file mode 100644 index 000000000..fcf869e19 --- /dev/null +++ b/zenserver/zenserver.vcxproj.filters @@ -0,0 +1,88 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <ClInclude Include="targetver.h" /> + <ClInclude Include="projectstore.h" /> + <ClInclude Include="casstore.h" /> + <ClInclude Include="vfs.h" /> + <ClInclude Include="testing\launch.h" /> + <ClInclude Include="cache\cacheagent.h"> + <Filter>cache</Filter> + </ClInclude> + <ClInclude Include="cache\cachestore.h"> + <Filter>cache</Filter> + </ClInclude> + <ClInclude Include="cache\kvcache.h"> + <Filter>cache</Filter> + </ClInclude> + <ClInclude Include="diag\diagsvcs.h"> + <Filter>diag</Filter> + </ClInclude> + <ClInclude Include="admin\admin.h"> + <Filter>admin</Filter> + </ClInclude> + <ClInclude Include="experimental\usnjournal.h"> + <Filter>experimental</Filter> + </ClInclude> + <ClInclude Include="upstream\jupiter.h"> + <Filter>upstream</Filter> + </ClInclude> + <ClInclude Include="upstream\zen.h"> + <Filter>upstream</Filter> + </ClInclude> + <ClInclude Include="cache\structuredcache.h"> + <Filter>cache</Filter> + </ClInclude> + <ClInclude Include="config.h" /> + <ClInclude Include="diag\logging.h" /> + <ClInclude Include="diag\crashreport.h" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="zenserver.cpp" /> + <ClCompile Include="projectstore.cpp" /> + <ClCompile Include="casstore.cpp" /> + <ClCompile Include="vfs.cpp" /> + <ClCompile Include="cache\cacheagent.cpp"> + <Filter>cache</Filter> + </ClCompile> + <ClCompile Include="cache\cachestore.cpp"> + <Filter>cache</Filter> + </ClCompile> + <ClCompile Include="experimental\usnjournal.cpp"> + <Filter>experimental</Filter> + </ClCompile> + <ClCompile Include="testing\launch.cpp" /> + <ClCompile Include="upstream\jupiter.cpp"> + <Filter>upstream</Filter> + </ClCompile> + <ClCompile Include="upstream\zen.cpp"> + <Filter>upstream</Filter> + </ClCompile> + <ClCompile Include="cache\structuredcache.cpp"> + <Filter>cache</Filter> + </ClCompile> + <ClCompile Include="cache\kvcache.cpp"> + <Filter>cache</Filter> + </ClCompile> + <ClCompile Include="config.cpp" /> + <ClCompile Include="diag\logging.cpp" /> + <ClCompile Include="diag\crashreport.cpp" /> + </ItemGroup> + <ItemGroup> + <Filter Include="cache"> + <UniqueIdentifier>{98e47c47-6bbe-46f5-b7cd-4b54352d964e}</UniqueIdentifier> + </Filter> + <Filter Include="diag"> + <UniqueIdentifier>{6a09a36e-fb5f-452a-ba0c-6d029240bad0}</UniqueIdentifier> + </Filter> + <Filter Include="admin"> + <UniqueIdentifier>{f72f861e-fa14-4ff8-9338-f0f84f4a8389}</UniqueIdentifier> + </Filter> + <Filter Include="experimental"> + <UniqueIdentifier>{76916270-97a6-4ec8-b323-a95b6080e245}</UniqueIdentifier> + </Filter> + <Filter Include="upstream"> + <UniqueIdentifier>{303c28c2-3607-4ef4-89bd-e3618fe37e74}</UniqueIdentifier> + </Filter> + </ItemGroup> +</Project>
\ No newline at end of file |