From 37edbbd4437bd756236d99684ac4d7b0b6725aa8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 20 May 2022 13:13:09 +0200 Subject: Automatically create namespaces on requests (if enabled via configuration) --- zenserver/cache/structuredcachestore.cpp | 71 +++++++++++++++++++++++--------- zenserver/cache/structuredcachestore.h | 11 ++++- zenserver/zenserver.cpp | 4 +- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index da948fd72..f582b61d8 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2130,12 +2130,16 @@ ZenCacheDiskLayer::TotalSize() const static constexpr std::string_view UE4DDCNamespaceName = "ue4.ddc"; -ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStorage(Gc), GcContributor(Gc) +ZenCacheStore::ZenCacheStore(CasGc& Gc, const Configuration& Configuration) +: GcStorage(Gc) +, GcContributor(Gc) +, m_Gc(Gc) +, m_Configuration(Configuration) { - CreateDirectories(BasePath); + CreateDirectories(m_Configuration.BasePath); DirectoryContent DirContent; - GetDirectoryContent(BasePath, DirectoryContent::IncludeDirsFlag, DirContent); + GetDirectoryContent(m_Configuration.BasePath, DirectoryContent::IncludeDirsFlag, DirContent); std::vector LegacyBuckets; std::vector Namespaces; @@ -2150,7 +2154,7 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor LegacyBuckets.push_back(DirName); } - ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), BasePath, LegacyBuckets.size()); + ZEN_INFO("Found #{} namespaces in '{}' and #{} legacy buckets", Namespaces.size(), m_Configuration.BasePath, LegacyBuckets.size()); if (std::find(Namespaces.begin(), Namespaces.end(), UE4DDCNamespaceName) == Namespaces.end()) { @@ -2158,13 +2162,14 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor ZEN_INFO("Moving #{} legacy buckets to '{}' namespace", LegacyBuckets.size(), UE4DDCNamespaceName); - std::filesystem::path DefaultNamespaceFolder = BasePath / fmt::format("{}{}", NamespaceDiskPrefix, UE4DDCNamespaceName); + std::filesystem::path DefaultNamespaceFolder = + m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, UE4DDCNamespaceName); CreateDirectories(DefaultNamespaceFolder); // Move any non-namespace folders into the default namespace folder for (const std::string& DirName : LegacyBuckets) { - std::filesystem::path LegacyFolder = BasePath / DirName; + std::filesystem::path LegacyFolder = m_Configuration.BasePath / DirName; std::filesystem::path NewPath = DefaultNamespaceFolder / DirName; std::error_code Ec; std::filesystem::rename(LegacyFolder, NewPath, Ec); @@ -2179,7 +2184,7 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor for (const std::string& NamespaceName : Namespaces) { m_Namespaces[NamespaceName] = - std::make_unique(Gc, BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName)); + std::make_unique(Gc, m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName)); } } @@ -2247,7 +2252,23 @@ ZenCacheStore::GetNamespace(std::string_view Namespace) return It->second.get(); } } - return nullptr; + _.ReleaseNow(); + + if (!m_Configuration.AllowAutomaticCreationOfNamespaces) + { + return nullptr; + } + + RwLock::ExclusiveLockScope __(m_NamespacesLock); + if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end()) + { + return It->second.get(); + } + + auto NewNamespace = m_Namespaces.insert_or_assign( + std::string(Namespace), + std::make_unique(m_Gc, m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, Namespace))); + return NewNamespace.first->second.get(); } void @@ -3048,40 +3069,52 @@ TEST_CASE("z$.namespaces") ScopedTemporaryDirectory TempDir; CreateDirectories(TempDir.Path()); + IoHash Key1; + IoHash Key2; { CasGc Gc; - ZenCacheStore Zcs(Gc, TempDir.Path() / "cache"); + ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = false}); const auto Bucket = "teardrinker"sv; const auto CustomNamespace = "mynamespace"sv; // Create a cache record - const IoHash Key = CreateKey(42); - CbObject CacheValue = CreateCacheValue(4096); + Key1 = CreateKey(42); + CbObject CacheValue = CreateCacheValue(4096); IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); Buffer.SetContentType(ZenContentType::kCbObject); ZenCacheValue PutValue = {.Value = Buffer}; - Zcs.Put(ZenCacheStore::DefaultNamespace, Bucket, Key, PutValue); + Zcs.Put(ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue); ZenCacheValue GetValue; - CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key, GetValue)); + CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue)); + CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue)); - CHECK(!Zcs.Get(CustomNamespace, Bucket, Key, GetValue)); + // This should just be dropped as we don't allow creating of namespaces on the fly + Zcs.Put(CustomNamespace, Bucket, Key1, PutValue); + CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue)); + } - // This should just be dropped for now until we decide how we add namespaces - Zcs.Put(CustomNamespace, Bucket, Key, PutValue); - CHECK(!Zcs.Get(CustomNamespace, Bucket, Key, GetValue)); + { + CasGc Gc; + ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}); + const auto Bucket = "teardrinker"sv; + const auto CustomNamespace = "mynamespace"sv; - const IoHash Key2 = CreateKey(43); - CbObject CacheValue2 = CreateCacheValue(4096); + Key2 = CreateKey(43); + CbObject CacheValue2 = CreateCacheValue(4096); IoBuffer Buffer2 = CacheValue2.GetBuffer().AsIoBuffer(); Buffer2.SetContentType(ZenContentType::kCbObject); ZenCacheValue PutValue2 = {.Value = Buffer2}; Zcs.Put(CustomNamespace, Bucket, Key2, PutValue2); + ZenCacheValue GetValue; CHECK(!Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key2, GetValue)); + CHECK(Zcs.Get(ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue)); + CHECK(!Zcs.Get(CustomNamespace, Bucket, Key1, GetValue)); + CHECK(Zcs.Get(CustomNamespace, Bucket, Key2, GetValue)); } } diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 34b8d18f4..13ba95f1b 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -360,7 +360,13 @@ public: "!default!"; // This is intentionally not a valid namespace name and will only be used for mapping when no namespace is given static constexpr std::string_view NamespaceDiskPrefix = "ns_"; - ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath); + struct Configuration + { + std::filesystem::path BasePath; + bool AllowAutomaticCreationOfNamespaces = true; + }; + + ZenCacheStore(CasGc& Gc, const Configuration& Configuration); ~ZenCacheStore(); bool Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); @@ -381,6 +387,9 @@ private: mutable RwLock m_NamespacesLock; NameSpaceMap m_Namespaces; + + CasGc& m_Gc; + Configuration m_Configuration; }; void z$_forcelink(); diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 9e6c67d34..b965a9bf3 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -755,7 +755,9 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) using namespace std::literals; ZEN_INFO("instantiating structured cache service"); - m_CacheStore = std::make_unique(m_CasGc, m_DataRoot / "cache"); + m_CacheStore = std::make_unique( + m_CasGc, + ZenCacheStore::Configuration{.BasePath = m_DataRoot / "cache", .AllowAutomaticCreationOfNamespaces = true}); const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; -- cgit v1.2.3 From f0d389a4430b62bfa8ea0852905fcf84065b08c2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 20 May 2022 18:45:35 +0200 Subject: Add catch2 support (#101) Added option to use catch2 for unit tests Currently both doctest and catch2 are supported via some compatibility macros. doctest is the default, and ZEN_USE_CATCH2 needs to be defined to switch to catch2. Our goal is to evaluate how well catch2 works and switch to catch2 if everything pans out since UE5 now supports using catch2 for unit tests. --- thirdparty/licenses/Catch2.tps | 13 +++++++++++++ thirdparty/licenses/Catch2_License.txt | 23 +++++++++++++++++++++++ xmake.lua | 1 + zen/zen.cpp | 5 ++--- zencore-test/xmake.lua | 1 + zencore-test/zencore-test.cpp | 5 ++--- zencore/blake3.cpp | 16 ++++++++-------- zencore/compactbinary.cpp | 4 ++-- zencore/compactbinarybuilder.cpp | 26 +++++++++++++------------- zencore/include/zencore/testing.h | 32 +++++++++++++++++++++++++++++++- zencore/md5.cpp | 16 ++++++++-------- zencore/sha1.cpp | 16 ++++++++-------- zencore/xmake.lua | 1 + zenserver-test/zenserver-test.cpp | 19 ++++++++++++++----- zenserver/zenserver.cpp | 5 ++--- zenstore-test/zenstore-test.cpp | 6 +++--- zenstore/blockstore.cpp | 6 ------ zenstore/compactcas.cpp | 6 +++++- zenstore/include/zenstore/blockstore.h | 2 ++ 19 files changed, 139 insertions(+), 64 deletions(-) create mode 100644 thirdparty/licenses/Catch2.tps create mode 100644 thirdparty/licenses/Catch2_License.txt diff --git a/thirdparty/licenses/Catch2.tps b/thirdparty/licenses/Catch2.tps new file mode 100644 index 000000000..e58e2cf3a --- /dev/null +++ b/thirdparty/licenses/Catch2.tps @@ -0,0 +1,13 @@ + + + Catch 2 + https://github.com/EpicGames/zen + Unit test framework. + https://github.com/catchorg/Catch2/blob/devel/LICENSE.txt + + Licensees + Git + P4 + + https://github.com/EpicGames/zen/tree/main/thirdparty/licenses + \ No newline at end of file diff --git a/thirdparty/licenses/Catch2_License.txt b/thirdparty/licenses/Catch2_License.txt new file mode 100644 index 000000000..127a5bc39 --- /dev/null +++ b/thirdparty/licenses/Catch2_License.txt @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/xmake.lua b/xmake.lua index 5ddcf7ce8..e4d795fcf 100644 --- a/xmake.lua +++ b/xmake.lua @@ -5,6 +5,7 @@ set_configvar("ZEN_SCHEMA_VERSION", 3) -- changed cas oplog format (p3rl) add_requires( "vcpkg::asio", + "vcpkg::catch2", "vcpkg::cpr", "vcpkg::curl", "vcpkg::cxxopts", diff --git a/zen/zen.cpp b/zen/zen.cpp index dc9be5d15..302e8496f 100644 --- a/zen/zen.cpp +++ b/zen/zen.cpp @@ -23,9 +23,8 @@ #include #if ZEN_WITH_TESTS -# define DOCTEST_CONFIG_IMPLEMENT +# define ZEN_TEST_WITH_RUNNER # include -# undef DOCTEST_CONFIG_IMPLEMENT #endif #include @@ -80,7 +79,7 @@ public: return GetLastError(); # endif // ZEN_PLATFORM_WINDOWS - return doctest::Context(argc, argv).run(); + return ZEN_RUN_TESTS(argc, argv); } virtual cxxopts::Options* Options() override { return &m_Options; } diff --git a/zencore-test/xmake.lua b/zencore-test/xmake.lua index 74c7e74a7..2556a1399 100644 --- a/zencore-test/xmake.lua +++ b/zencore-test/xmake.lua @@ -5,4 +5,5 @@ target("zencore-test") add_headerfiles("**.h") add_files("*.cpp") add_deps("zencore") + add_packages("vcpkg::catch2") add_packages("vcpkg::doctest") diff --git a/zencore-test/zencore-test.cpp b/zencore-test/zencore-test.cpp index 7d7365c54..53413fb25 100644 --- a/zencore-test/zencore-test.cpp +++ b/zencore-test/zencore-test.cpp @@ -7,9 +7,8 @@ #include #if ZEN_WITH_TESTS -# define DOCTEST_CONFIG_IMPLEMENT +# define ZEN_TEST_WITH_RUNNER 1 # include -# undef DOCTEST_CONFIG_IMPLEMENT #endif int @@ -20,7 +19,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) zen::logging::InitializeLogging(); - return doctest::Context(argc, argv).run(); + return ZEN_RUN_TESTS(argc, argv); #else return 0; #endif diff --git a/zencore/blake3.cpp b/zencore/blake3.cpp index b2019d4e2..02d6eb241 100644 --- a/zencore/blake3.cpp +++ b/zencore/blake3.cpp @@ -127,14 +127,14 @@ BLAKE3Stream::GetHash() #if ZEN_WITH_TESTS -doctest::String -toString(const BLAKE3& value) -{ - char text[2 * sizeof(BLAKE3) + 1]; - value.ToHexString(text); - - return text; -} +// doctest::String +// toString(const BLAKE3& value) +// { +// char text[2 * sizeof(BLAKE3) + 1]; +// value.ToHexString(text); + +// return text; +// } TEST_CASE("BLAKE3") { diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index a51253989..375a97fc5 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -2133,8 +2133,8 @@ TEST_CASE("uson.json") const double DoubleValue = Json["Double"].number_value(); CHECK(JsonError.empty()); - CHECK(FloatValue == doctest::Approx(ExpectedFloatValue)); - CHECK(DoubleValue == doctest::Approx(ExpectedDoubleValue)); + CHECK(FloatValue == Approx(ExpectedFloatValue)); + CHECK(DoubleValue == Approx(ExpectedDoubleValue)); } SUBCASE("number.nan") diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp index 1d2ba45df..d4ccd434d 100644 --- a/zencore/compactbinarybuilder.cpp +++ b/zencore/compactbinarybuilder.cpp @@ -706,19 +706,19 @@ usonbuilder_forcelink() { } -doctest::String -toString(const DateTime&) -{ - // TODO:implement - return ""; -} - -doctest::String -toString(const TimeSpan&) -{ - // TODO:implement - return ""; -} +// doctest::String +// toString(const DateTime&) +// { +// // TODO:implement +// return ""; +// } + +// doctest::String +// toString(const TimeSpan&) +// { +// // TODO:implement +// return ""; +// } TEST_CASE("usonbuilder.object") { diff --git a/zencore/include/zencore/testing.h b/zencore/include/zencore/testing.h index 80aebc26e..faa4eef6c 100644 --- a/zencore/include/zencore/testing.h +++ b/zencore/include/zencore/testing.h @@ -4,6 +4,36 @@ #include +#ifdef ZEN_TEST_WITH_RUNNER +# define CATCH_CONFIG_RUNNER +# define DOCTEST_CONFIG_IMPLEMENT +#endif + +#ifndef ZEN_USE_CATCH2 +# define ZEN_USE_CATCH2 0 +#endif + #if ZEN_WITH_TESTS -# include +# if ZEN_USE_CATCH2 +# include +# define SUBCASE(x) SECTION(x) +# define CHECK_EQ(x, y) CHECK((x) == (y)) +# define CHECK_MESSAGE(x, y) CHECK(x) +# define ZEN_RUN_TESTS(argC, argV) Catch::Session().run(argC, argV) +# else +# include +# define ZEN_RUN_TESTS(argC, argV) doctest::Context(argc, argv).run() +inline auto +Approx(auto Value) +{ + return doctest::Approx(Value); +} +# endif +#else +# define ZEN_RUN_TESTS(argC, argV) +#endif + +#ifdef ZEN_TEST_WITH_RUNNER +# undef CATCH_CONFIG_RUNNER +# undef DOCTEST_CONFIG_IMPLEMENT #endif diff --git a/zencore/md5.cpp b/zencore/md5.cpp index faece3862..4ec145697 100644 --- a/zencore/md5.cpp +++ b/zencore/md5.cpp @@ -428,14 +428,14 @@ md5_forcelink() { } -doctest::String -toString(const MD5& value) -{ - char md5text[2 * sizeof(MD5) + 1]; - value.ToHexString(md5text); - - return md5text; -} +// doctest::String +// toString(const MD5& value) +// { +// char md5text[2 * sizeof(MD5) + 1]; +// value.ToHexString(md5text); + +// return md5text; +// } TEST_CASE("MD5") { diff --git a/zencore/sha1.cpp b/zencore/sha1.cpp index 66e01f232..3ee74d7d8 100644 --- a/zencore/sha1.cpp +++ b/zencore/sha1.cpp @@ -364,14 +364,14 @@ sha1_forcelink() { } -doctest::String -toString(const SHA1& value) -{ - char sha1text[2 * sizeof(SHA1) + 1]; - value.ToHexString(sha1text); - - return sha1text; -} +// doctest::String +// toString(const SHA1& value) +// { +// char sha1text[2 * sizeof(SHA1) + 1]; +// value.ToHexString(sha1text); + +// return sha1text; +// } TEST_CASE("SHA1") { diff --git a/zencore/xmake.lua b/zencore/xmake.lua index 1044c5025..63e874ac5 100644 --- a/zencore/xmake.lua +++ b/zencore/xmake.lua @@ -33,6 +33,7 @@ target('zencore') "vcpkg::spdlog", "vcpkg::fmt", "vcpkg::doctest", + "vcpkg::catch2", "vcpkg::json11", "vcpkg::lz4", "vcpkg::mimalloc", diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index a6e89e702..21837fc0a 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -65,9 +65,8 @@ ZEN_THIRD_PARTY_INCLUDES_END ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS -# define DOCTEST_CONFIG_IMPLEMENT +# define ZEN_TEST_WITH_RUNNER 1 # include -# undef DOCTEST_CONFIG_IMPLEMENT #endif using namespace std::literals; @@ -331,7 +330,8 @@ main(int argc, char** argv) TestEnv.InitializeForTest(ProgramBaseDir, TestBaseDir); ZEN_INFO("Running tests...(base dir: '{}')", TestBaseDir); - return doctest::Context(argc, argv).run(); + + return ZEN_RUN_TESTS(argc, argv); } namespace zen::tests { @@ -2504,8 +2504,12 @@ private: zen::BinaryWriter m_MemOut; }; -TEST_CASE("exec.basic" * doctest::skip(true)) +TEST_CASE(".exec.basic" /* * doctest::skip(true) */) { + if (true) + { + return; + } # if ZEN_WITH_EXEC_SERVICES using namespace std::literals; @@ -2733,8 +2737,13 @@ TEST_CASE("http.package") CHECK_EQ(ResponsePackage, TestPackage); } -TEST_CASE("websocket.basic" * doctest::skip(true)) +TEST_CASE("websocket.basic") { + if (true) + { + return; + } + std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); const uint16_t PortNumber = 13337; const auto MaxWaitTime = std::chrono::seconds(5); diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 9e6c67d34..05ae4a09a 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -50,9 +50,8 @@ ZEN_THIRD_PARTY_INCLUDES_END // in some shared code into the executable #if ZEN_WITH_TESTS -# define DOCTEST_CONFIG_IMPLEMENT +# define ZEN_TEST_WITH_RUNNER # include -# undef DOCTEST_CONFIG_IMPLEMENT #endif ////////////////////////////////////////////////////////////////////////// @@ -1162,7 +1161,7 @@ test_main(int argc, char** argv) zen::MaximizeOpenFileCount(); - return doctest::Context(argc, argv).run(); + return ZEN_RUN_TESTS(argc, argv); } #endif diff --git a/zenstore-test/zenstore-test.cpp b/zenstore-test/zenstore-test.cpp index ebc0806d5..00c1136b6 100644 --- a/zenstore-test/zenstore-test.cpp +++ b/zenstore-test/zenstore-test.cpp @@ -12,9 +12,8 @@ #endif #if ZEN_WITH_TESTS -# define DOCTEST_CONFIG_IMPLEMENT +# define ZEN_TEST_WITH_RUNNER 1 # include -# undef DOCTEST_CONFIG_IMPLEMENT #endif int @@ -26,7 +25,8 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); - return doctest::Context(argc, argv).run(); + return ZEN_RUN_TESTS(argc, argv); #else + return 0; #endif } diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp index d490678b5..3e27b0d12 100644 --- a/zenstore/blockstore.cpp +++ b/zenstore/blockstore.cpp @@ -911,12 +911,6 @@ BlockStore::GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint #if ZEN_WITH_TESTS -static bool -operator==(const BlockStoreLocation& Lhs, const BlockStoreLocation& Rhs) -{ - return Lhs.BlockIndex == Rhs.BlockIndex && Lhs.Offset == Rhs.Offset && Lhs.Size == Rhs.Size; -} - TEST_CASE("blockstore.blockstoredisklocation") { BlockStoreLocation Zero = BlockStoreLocation{.BlockIndex = 0, .Offset = 0, .Size = 0}; diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 2d48265f7..65f959a0e 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -1894,8 +1894,12 @@ TEST_CASE("compactcas.threadedinsert") } } -TEST_CASE("compactcas.migrate.large.data" * doctest::skip(true)) +TEST_CASE("compactcas.migrate.large.data") // * doctest::skip(true)) { + if (true) + { + return; + } const char* BigDataPath = "D:\\zen-data\\dc4-zen-cache-t\\cas"; std::filesystem::path TobsBasePath = GetBasePath(BigDataPath, "tobs"); std::filesystem::path SobsBasePath = GetBasePath(BigDataPath, "sobs"); diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h index 34c475fb6..000395fb9 100644 --- a/zenstore/include/zenstore/blockstore.h +++ b/zenstore/include/zenstore/blockstore.h @@ -64,6 +64,8 @@ struct BlockStoreDiskLocation inline uint64_t GetSize() const { return m_Size; } + inline auto operator<=>(const BlockStoreDiskLocation& Rhs) const = default; + private: inline void Init(uint32_t BlockIndex, uint64_t Offset, uint64_t Size) { -- cgit v1.2.3 From dba8b362221188c4e3125e39eb0654613a7d3dd2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 23 May 2022 14:44:48 +0200 Subject: De/fix namespace folder scanning (#103) --- zenserver/cache/structuredcachestore.cpp | 2 +- zenstore/gc.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index da948fd72..7509c5a71 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2141,7 +2141,7 @@ ZenCacheStore::ZenCacheStore(CasGc& Gc, std::filesystem::path BasePath) : GcStor std::vector Namespaces; for (const std::filesystem::path& DirPath : DirContent.Directories) { - std::string DirName = PathToUtf8(DirPath.stem()); + std::string DirName = PathToUtf8(DirPath.filename()); if (DirName.starts_with(NamespaceDiskPrefix)) { Namespaces.push_back(DirName.substr(NamespaceDiskPrefix.length())); diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 4b50668d9..8e2d441f8 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -343,7 +343,7 @@ GcStorage::GcStorage(CasGc& Gc) : m_Gc(Gc) GcStorage::~GcStorage() { - m_Gc.AddGcStorage(this); + m_Gc.RemoveGcStorage(this); } ////////////////////////////////////////////////////////////////////////// @@ -373,6 +373,7 @@ CasGc::RemoveGcContributor(GcContributor* Contributor) void CasGc::AddGcStorage(GcStorage* Storage) { + ZEN_ASSERT(Storage != nullptr); RwLock::ExclusiveLockScope _(m_Lock); m_GcStorage.push_back(Storage); } -- cgit v1.2.3 From 4e98a42e43a1c03678d7dbbed1aeb583879fe677 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Mon, 23 May 2022 10:06:32 -0700 Subject: Update README to direct to CODING for contributing (#104) --- README.md | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/README.md b/README.md index f8cc6779a..d3fc465dd 100644 --- a/README.md +++ b/README.md @@ -194,12 +194,7 @@ xmake build # Contributing Code -To run the pre-commit scripts you'll need a few things: - -* We rely on clang-format for consistent code formatting. You can install version 12 or later from https://llvm.org/builds/ -* The helper scripts also depend on Python 3.x, which you may install from https://www.python.org/downloads/windows/ (I am presently using 3.9.5 which works). NOTE: *do* check the option to add Python to your PATH! - -Once you have those dependencies, you can simply run `prepare_commit.bat` to ensure the code is properly formatted and has the Epic copyright header comment. I'm sure there's a better way to integrate this into the git submit flow but my git-fu is not strong enough yet to know how to do that best. +See [CODING.md](CODING.md) # Debugging @@ -223,12 +218,3 @@ Registering a handler for an HTTP endpoint requires either process elevation (i. or `netsh http add urlacl url=http://*:1337/ sddl=D:(A;;GX;;;S-1-1-0)` (enable for any authenticated user) - -# Coding Standards - -See [CODING.md](CODING.md) - -Run `prepare_commit.bat` before committing code. It ensures all source files are formatted with -clang-format which you will need to install. - -(More helpful instructions needed here :) -- cgit v1.2.3 From 898a48b5121ed09879e34dc3315387d927a91ebf Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 13:24:51 +0200 Subject: Make sure to hold exclusive lock over index and all shard locks. Clear index on drop. --- zenserver/cache/structuredcachestore.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 3189a14cc..1db99280e 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1199,8 +1199,16 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& void ZenCacheDiskLayer::CacheBucket::Drop() { + RwLock::ExclusiveLockScope _(m_IndexLock); + std::vector> ShardLocks; + ShardLocks.reserve(256); + for (RwLock& Lock : m_ShardedLocks) + { + ShardLocks.push_back(std::make_unique(Lock)); + } m_BlockStore.Close(); m_SlogFile.Close(); + m_Index.clear(); DeleteDirectories(m_BucketDir); } @@ -2274,7 +2282,7 @@ ZenCacheStore::GetNamespace(std::string_view Namespace) void ZenCacheStore::IterateNamespaces(const std::function& Callback) const { - std::vector > Namespaces; + std::vector> Namespaces; { RwLock::SharedLockScope _(m_NamespacesLock); Namespaces.reserve(m_Namespaces.size()); -- cgit v1.2.3 From ba135d4a90746ed905a825f48fc3ccfe0641301e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 13:52:45 +0200 Subject: Use rename/delete and keep pointer for dropped buckets --- zenserver/cache/structuredcachestore.cpp | 157 ++++++++++++++++++++----------- zenserver/cache/structuredcachestore.h | 49 +++++----- 2 files changed, 129 insertions(+), 77 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 1db99280e..be9e408b4 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -580,19 +580,6 @@ ZenCacheDiskLayer::CacheBucket::~CacheBucket() { } -bool -ZenCacheDiskLayer::CacheBucket::Delete(std::filesystem::path BucketDir) -{ - if (std::filesystem::exists(BucketDir)) - { - DeleteDirectories(BucketDir); - - return true; - } - - return false; -} - void ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate) { @@ -1158,11 +1145,6 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, bool ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue) { - if (!m_IsOk) - { - return false; - } - RwLock::SharedLockScope _(m_IndexLock); auto It = m_Index.find(HashKey); if (It == m_Index.end()) @@ -1184,11 +1166,6 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal void ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value) { - if (!m_IsOk) - { - return; - } - if (Value.Value.Size() >= m_LargeObjectThreshold) { return PutStandaloneCacheValue(HashKey, Value); @@ -1196,10 +1173,43 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& PutInlineCacheValue(HashKey, Value); } +static bool +DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view BucketName) +{ + int dropIndex = 0; + do + { + if (!std::filesystem::exists(BucketDir)) + { + return false; + } + + std::string DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, dropIndex); + std::filesystem::path DroppedBucketPath = BucketDir.parent_path() / DroppedBucketName; + if (std::filesystem::exists(DroppedBucketPath)) + { + dropIndex++; + continue; + } + + std::error_code Ec; + std::filesystem::rename(BucketDir, DroppedBucketPath, Ec); + if (!Ec) + { + DeleteDirectories(DroppedBucketPath); + return true; + } + // TODO: Do we need to bail at some point? + zen::Sleep(100); + } while (true); +} + void ZenCacheDiskLayer::CacheBucket::Drop() { - RwLock::ExclusiveLockScope _(m_IndexLock); + RwLock::ExclusiveLockScope _(m_IndexLock); + m_IsOk = false; + std::vector> ShardLocks; ShardLocks.reserve(256); for (RwLock& Lock : m_ShardedLocks) @@ -1208,8 +1218,10 @@ ZenCacheDiskLayer::CacheBucket::Drop() } m_BlockStore.Close(); m_SlogFile.Close(); + + DeleteBucketFromDisk(m_BucketDir, m_BucketName); + m_Index.clear(); - DeleteDirectories(m_BucketDir); } void @@ -1711,7 +1723,12 @@ ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx) for (auto& Kv : m_Buckets) { - Kv.second.CollectGarbage(GcCtx); + CacheBucket& Bucket = *Kv.second; + if (!Bucket.IsOk()) + { + continue; + } + Bucket.CollectGarbage(GcCtx); } } @@ -1724,7 +1741,11 @@ ZenCacheDiskLayer::UpdateAccessTimes(const zen::access_tracking::AccessTimes& Ac { if (auto It = m_Buckets.find(Kv.first); It != m_Buckets.end()) { - CacheBucket& Bucket = It->second; + CacheBucket& Bucket = *It->second; + if (!Bucket.IsOk()) + { + continue; + } Bucket.UpdateAccessTimes(Kv.second); } } @@ -1937,7 +1958,11 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach if (it != m_Buckets.end()) { - Bucket = &it->second; + Bucket = it->second.get(); + if (!Bucket->IsOk()) + { + return false; + } } } @@ -1949,22 +1974,25 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end()) { - Bucket = &it->second; + Bucket = it->second.get(); } else { - auto It = m_Buckets.try_emplace(BucketName, BucketName); - Bucket = &It.first->second; + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); + Bucket = InsertResult.first->second.get(); std::filesystem::path BucketPath = m_RootDir; BucketPath /= BucketName; Bucket->OpenOrCreate(BucketPath); + if (!Bucket->IsOk()) + { + return false; + } } } ZEN_ASSERT(Bucket != nullptr); - return Bucket->Get(HashKey, OutValue); } @@ -1981,7 +2009,11 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z if (it != m_Buckets.end()) { - Bucket = &it->second; + Bucket = it->second.get(); + if (!Bucket->IsOk()) + { + return; + } } } @@ -1993,26 +2025,27 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end()) { - Bucket = &it->second; + Bucket = it->second.get(); } else { - auto It = m_Buckets.try_emplace(BucketName, BucketName); - Bucket = &It.first->second; + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); + Bucket = InsertResult.first->second.get(); std::filesystem::path BucketPath = m_RootDir; BucketPath /= BucketName; Bucket->OpenOrCreate(BucketPath); + if (!Bucket->IsOk()) + { + return; + } } } ZEN_ASSERT(Bucket != nullptr); - if (Bucket->IsOk()) - { - Bucket->Put(HashKey, Value); - } + Bucket->Put(HashKey, Value); } void @@ -2034,9 +2067,8 @@ ZenCacheDiskLayer::DiscoverBuckets() } else { - auto InsertResult = m_Buckets.try_emplace(BucketName, BucketName); - - CacheBucket& Bucket = InsertResult.first->second; + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); + CacheBucket& Bucket = *InsertResult.first->second; Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false); @@ -2063,19 +2095,23 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) if (it != m_Buckets.end()) { - CacheBucket* Bucket = &it->second; - - Bucket->Drop(); - + CacheBucket& Bucket = *it->second; + m_DroppedBuckets.push_back(std::move(it->second)); m_Buckets.erase(it); + if (!Bucket.IsOk()) + { + return false; + } + + Bucket.Drop(); return true; } + // Make sure we remove the folder even if we don't know about the bucket std::filesystem::path BucketPath = m_RootDir; BucketPath /= std::string(InBucket); - - return CacheBucket::Delete(BucketPath); + return DeleteBucketFromDisk(BucketPath, InBucket); } void @@ -2088,7 +2124,12 @@ ZenCacheDiskLayer::Flush() Buckets.reserve(m_Buckets.size()); for (auto& Kv : m_Buckets) { - Buckets.push_back(&Kv.second); + CacheBucket* Bucket = Kv.second.get(); + if (!Bucket->IsOk()) + { + continue; + } + Buckets.push_back(Bucket); } } @@ -2105,7 +2146,12 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) for (auto& Kv : m_Buckets) { - Kv.second.Scrub(Ctx); + CacheBucket& Bucket = *Kv.second; + if (!Bucket.IsOk()) + { + continue; + } + Bucket.Scrub(Ctx); } } @@ -2116,7 +2162,12 @@ ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx) for (auto& Kv : m_Buckets) { - Kv.second.GatherReferences(GcCtx); + CacheBucket& Bucket = *Kv.second; + if (!Bucket.IsOk()) + { + continue; + } + Bucket.GatherReferences(GcCtx); } } @@ -2128,7 +2179,7 @@ ZenCacheDiskLayer::TotalSize() const for (auto& Kv : m_Buckets) { - TotalSize += Kv.second.TotalSize(); + TotalSize += Kv.second->TotalSize(); } return TotalSize; diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 13ba95f1b..bae15231b 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -233,17 +233,15 @@ private: CacheBucket(std::string BucketName); ~CacheBucket(); - void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true); - static bool Delete(std::filesystem::path BucketDir); - bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); - void Put(const IoHash& HashKey, const ZenCacheValue& Value); - void Drop(); - void Flush(); - void SaveManifest(); - void Scrub(ScrubContext& Ctx); - void GatherReferences(GcContext& GcCtx); - void CollectGarbage(GcContext& GcCtx); - void UpdateAccessTimes(const std::vector& AccessTimes); + void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true); + bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(const IoHash& HashKey, const ZenCacheValue& Value); + void Drop(); + void Flush(); + void Scrub(ScrubContext& Ctx); + void GatherReferences(GcContext& GcCtx); + void CollectGarbage(GcContext& GcCtx); + void UpdateAccessTimes(const std::vector& AccessTimes); inline bool IsOk() const { return m_IsOk; } inline uint64_t TotalSize() const { return m_TotalSize.load(std::memory_order::relaxed); } @@ -292,16 +290,18 @@ private: std::atomic_uint64_t m_TotalSize{}; - void BuildPath(PathBuilderBase& Path, const IoHash& HashKey); - void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); - bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue); - void PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); - bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); - void MakeIndexSnapshot(); - uint64_t ReadIndexFile(); - uint64_t ReadLog(uint64_t LogPosition); - uint64_t MigrateLegacyData(bool CleanSource); - void OpenLog(const std::filesystem::path& BucketDir, const bool IsNew); + void BuildPath(PathBuilderBase& Path, const IoHash& HashKey); + void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); + bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue); + void PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); + bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); + void MakeIndexSnapshot(); + uint64_t ReadIndexFile(); + uint64_t ReadLog(uint64_t LogPosition); + uint64_t MigrateLegacyData(bool CleanSource); + void OpenLog(const std::filesystem::path& BucketDir, const bool IsNew); + static bool Delete(std::filesystem::path BucketDir); + void SaveManifest(); // These locks are here to avoid contention on file creation, therefore it's sufficient // that we take the same lock for the same hash @@ -314,9 +314,10 @@ private: inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; } }; - std::filesystem::path m_RootDir; - mutable RwLock m_Lock; - std::unordered_map m_Buckets; // TODO: make this case insensitive + std::filesystem::path m_RootDir; + mutable RwLock m_Lock; + std::unordered_map> m_Buckets; // TODO: make this case insensitive + std::vector> m_DroppedBuckets; ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete; ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete; -- cgit v1.2.3 From 4d1faaa2076bff77734d152b1403c3c90884bc83 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 23:22:52 +0200 Subject: drop bucket test --- zenserver/cache/structuredcachestore.cpp | 73 ++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index be9e408b4..f3f6503f3 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -3177,6 +3177,79 @@ TEST_CASE("z$.namespaces") } } +TEST_CASE("z$.drop.bucket") +{ + using namespace testutils; + + const auto CreateCacheValue = [](size_t Size) -> CbObject { + std::vector Buf; + Buf.resize(Size); + + CbObjectWriter Writer; + Writer.AddBinary("Binary"sv, Buf.data(), Buf.size()); + return Writer.Save(); + }; + + ScopedTemporaryDirectory TempDir; + CreateDirectories(TempDir.Path()); + + IoHash Key1; + IoHash Key2; + + auto PutValue = + [&CreateCacheValue](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, size_t KeyIndex, size_t Size) { + // Create a cache record + IoHash Key = CreateKey(KeyIndex); + CbObject CacheValue = CreateCacheValue(Size); + + IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); + Buffer.SetContentType(ZenContentType::kCbObject); + + ZenCacheValue PutValue = {.Value = Buffer}; + Zcs.Put(Namespace, Bucket, Key, PutValue); + return Key; + }; + auto GetValue = [](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) { + ZenCacheValue GetValue; + Zcs.Get(Namespace, Bucket, Key, GetValue); + return GetValue; + }; + WorkerThreadPool Workers(1); + { + CasGc Gc; + ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}); + const auto Bucket = "teardrinker"sv; + const auto Namespace = "mynamespace"sv; + + Key1 = PutValue(Zcs, Namespace, Bucket, 42, 4096); + Key2 = PutValue(Zcs, Namespace, Bucket, 43, 2048); + + ZenCacheValue Value1 = GetValue(Zcs, Namespace, Bucket, Key1); + CHECK(Value1.Value); + + std::atomic_bool WorkComplete = false; + Workers.ScheduleWork([&]() { + zen::Sleep(100); + Value1.Value = IoBuffer{}; + Value1 = GetValue(Zcs, Namespace, Bucket, Key1); + CHECK(!Value1.Value); + WorkComplete = true; + }); + // On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket + // Our DropBucket execution blocks any incoming request from completing until we are done with the drop + Zcs.DropBucket(Namespace, Bucket); + while (!WorkComplete) + { + zen::Sleep(1); + } + CHECK(!Value1.Value); + + // Entire bucket should be dropped, but doing a request should will re-create the namespace but it must still be empty + ZenCacheValue Value2 = GetValue(Zcs, Namespace, Bucket, Key2); + CHECK(!Value2.Value); + } +} + TEST_CASE("z$.blocked.disklayer.put") { ScopedTemporaryDirectory TempDir; -- cgit v1.2.3 From a5c70af49a66c0a708ba612a29e3cd4639e1c613 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 23:33:02 +0200 Subject: run clang format on single instance, don't wait for clang-format --- .github/workflows/self_host_build.yml | 75 +++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/.github/workflows/self_host_build.yml b/.github/workflows/self_host_build.yml index 3864151ce..eb378e5f6 100644 --- a/.github/workflows/self_host_build.yml +++ b/.github/workflows/self_host_build.yml @@ -9,30 +9,72 @@ jobs: clang-format: name: Check clang-format runs-on: [self-hosted, linux, x64] - strategy: - matrix: - path: - - 'zen' - - 'zencore' - - 'zencore-test' - - 'zenhttp' - - 'zenserver-test' - - 'zenstore' - - 'zenstore-test' - - 'zentest-appstub' - - 'zenutil' - - 'zenserver' + steps: - uses: actions/checkout@v2 - - name: clang-format ${{ matrix.path }} + + - name: clang-format 'zen' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zen' + + - name: clang-format 'zencore' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zencore' + + - name: clang-format 'zencore-test' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zencore-test' + + - name: clang-format 'zenhttp' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zenhttp' + + - name: clang-format 'zenserver-test' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zenserver-test' + + - name: clang-format 'zenstore' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zenstore' + + - name: clang-format 'zenstore-test' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zenstore-test' + + - name: clang-format 'zentest-appstub' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zentest-appstub' + + - name: clang-format 'zenutil' + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: 'zenutil' + + - name: clang-format 'zenserver' uses: jidicula/clang-format-action@v4.6.2 with: clang-format-version: '13' - check-path: ${{ matrix.path }} + check-path: 'zenserver' windows-build: name: Build Windows - needs: clang-format runs-on: [self-hosted, windows, x64] strategy: matrix: @@ -81,7 +123,6 @@ jobs: linux-build: name: Build Linux - needs: clang-format runs-on: [self-hosted, linux, x64] strategy: matrix: -- cgit v1.2.3 From 0fa48b0dcb6bcd5d1ea3d34a8d3f7e39951a7282 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 23:52:45 +0200 Subject: use exclude regex in clang format --- .github/workflows/self_host_build.yml | 261 ++++++++++++++-------------------- 1 file changed, 104 insertions(+), 157 deletions(-) diff --git a/.github/workflows/self_host_build.yml b/.github/workflows/self_host_build.yml index eb378e5f6..1c7b466ca 100644 --- a/.github/workflows/self_host_build.yml +++ b/.github/workflows/self_host_build.yml @@ -13,163 +13,110 @@ jobs: steps: - uses: actions/checkout@v2 - - name: clang-format 'zen' + - name: clang-format uses: jidicula/clang-format-action@v4.6.2 with: clang-format-version: '13' - check-path: 'zen' - - - name: clang-format 'zencore' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zencore' - - - name: clang-format 'zencore-test' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zencore-test' - - - name: clang-format 'zenhttp' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zenhttp' - - - name: clang-format 'zenserver-test' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zenserver-test' - - - name: clang-format 'zenstore' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zenstore' - - - name: clang-format 'zenstore-test' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zenstore-test' - - - name: clang-format 'zentest-appstub' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zentest-appstub' - - - name: clang-format 'zenutil' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zenutil' - - - name: clang-format 'zenserver' - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: 'zenserver' - - windows-build: - name: Build Windows - runs-on: [self-hosted, windows, x64] - strategy: - matrix: - config: - - 'debug' - - 'release' - arch: - - 'x64' - env: - VCPKG_VERSION: 2022.03.10 - - steps: - - uses: actions/checkout@v2 - - - name: Setup xmake - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: 2.6.4 - - - name: Installing vcpkg - run: | - git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg - cd .vcpkg - .\bootstrap-vcpkg.bat - .\vcpkg.exe integrate install - cd .. - - - name: Cache vcpkg - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}\.vcpkg\installed - key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - - - name: Config - run: | - xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: Build & Test - run: | - xmake test -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - linux-build: - name: Build Linux - runs-on: [self-hosted, linux, x64] - strategy: - matrix: - config: - - 'debug' - - 'release' - arch: - - 'x86_64' - env: - VCPKG_VERSION: 2022.03.10 - - steps: - - uses: actions/checkout@v2 - - - name: Set up GCC 11 - uses: egor-tensin/setup-gcc@v1 - with: - version: 11 - platform: x64 - - - name: Setup xmake - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: 2.6.4 - - - name: Installing vcpkg - run: | - git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg - cd .vcpkg - ./bootstrap-vcpkg.sh - cd .. - - - name: Cache vcpkg - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/.vcpkg/installed - key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - - - name: Config - run: | - xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: Build & Test - run: | - xmake test -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + check-path: '.' + exclude-regex: (.*thirdparty.*) + +# windows-build: +# name: Build Windows +# runs-on: [self-hosted, windows, x64] +# strategy: +# matrix: +# config: +# - 'debug' +# - 'release' +# arch: +# - 'x64' +# env: +# VCPKG_VERSION: 2022.03.10 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Setup xmake +# uses: xmake-io/github-action-setup-xmake@v1 +# with: +# xmake-version: 2.6.4 +# +# - name: Installing vcpkg +# run: | +# git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg +# cd .vcpkg +# .\bootstrap-vcpkg.bat +# .\vcpkg.exe integrate install +# cd .. +# +# - name: Cache vcpkg +# uses: actions/cache@v2 +# with: +# path: | +# ${{ github.workspace }}\.vcpkg\installed +# key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 +# +# - name: Config +# run: | +# xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} +# env: +# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg +# +# - name: Build & Test +# run: | +# xmake test -v -y +# env: +# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg +# +# linux-build: +# name: Build Linux +# runs-on: [self-hosted, linux, x64] +# strategy: +# matrix: +# config: +# - 'debug' +# - 'release' +# arch: +# - 'x86_64' +# env: +# VCPKG_VERSION: 2022.03.10 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Set up GCC 11 +# uses: egor-tensin/setup-gcc@v1 +# with: +# version: 11 +# platform: x64 +# +# - name: Setup xmake +# uses: xmake-io/github-action-setup-xmake@v1 +# with: +# xmake-version: 2.6.4 +# +# - name: Installing vcpkg +# run: | +# git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg +# cd .vcpkg +# ./bootstrap-vcpkg.sh +# cd .. +# +# - name: Cache vcpkg +# uses: actions/cache@v2 +# with: +# path: | +# ${{ github.workspace }}/.vcpkg/installed +# key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 +# +# - name: Config +# run: | +# xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} +# env: +# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg +# +# - name: Build & Test +# run: | +# xmake test -v -y +# env: +# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -- cgit v1.2.3 From 99de2b0b94303b164f08ff095f3284f013b5439c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 23:56:59 +0200 Subject: re-enable tests --- .github/workflows/self_host_build.yml | 200 +++++++++++++++++----------------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/.github/workflows/self_host_build.yml b/.github/workflows/self_host_build.yml index 1c7b466ca..8bb57a4b4 100644 --- a/.github/workflows/self_host_build.yml +++ b/.github/workflows/self_host_build.yml @@ -20,103 +20,103 @@ jobs: check-path: '.' exclude-regex: (.*thirdparty.*) -# windows-build: -# name: Build Windows -# runs-on: [self-hosted, windows, x64] -# strategy: -# matrix: -# config: -# - 'debug' -# - 'release' -# arch: -# - 'x64' -# env: -# VCPKG_VERSION: 2022.03.10 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Setup xmake -# uses: xmake-io/github-action-setup-xmake@v1 -# with: -# xmake-version: 2.6.4 -# -# - name: Installing vcpkg -# run: | -# git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg -# cd .vcpkg -# .\bootstrap-vcpkg.bat -# .\vcpkg.exe integrate install -# cd .. -# -# - name: Cache vcpkg -# uses: actions/cache@v2 -# with: -# path: | -# ${{ github.workspace }}\.vcpkg\installed -# key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 -# -# - name: Config -# run: | -# xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} -# env: -# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -# -# - name: Build & Test -# run: | -# xmake test -v -y -# env: -# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -# -# linux-build: -# name: Build Linux -# runs-on: [self-hosted, linux, x64] -# strategy: -# matrix: -# config: -# - 'debug' -# - 'release' -# arch: -# - 'x86_64' -# env: -# VCPKG_VERSION: 2022.03.10 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Set up GCC 11 -# uses: egor-tensin/setup-gcc@v1 -# with: -# version: 11 -# platform: x64 -# -# - name: Setup xmake -# uses: xmake-io/github-action-setup-xmake@v1 -# with: -# xmake-version: 2.6.4 -# -# - name: Installing vcpkg -# run: | -# git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg -# cd .vcpkg -# ./bootstrap-vcpkg.sh -# cd .. -# -# - name: Cache vcpkg -# uses: actions/cache@v2 -# with: -# path: | -# ${{ github.workspace }}/.vcpkg/installed -# key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 -# -# - name: Config -# run: | -# xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} -# env: -# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -# -# - name: Build & Test -# run: | -# xmake test -v -y -# env: -# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + windows-build: + name: Build Windows + runs-on: [self-hosted, windows, x64] + strategy: + matrix: + config: + - 'debug' + - 'release' + arch: + - 'x64' + env: + VCPKG_VERSION: 2022.03.10 + + steps: + - uses: actions/checkout@v2 + + - name: Setup xmake + uses: xmake-io/github-action-setup-xmake@v1 + with: + xmake-version: 2.6.4 + + - name: Installing vcpkg + run: | + git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg + cd .vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg.exe integrate install + cd .. + + - name: Cache vcpkg + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}\.vcpkg\installed + key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + + - name: Config + run: | + xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: Build & Test + run: | + xmake test -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + linux-build: + name: Build Linux + runs-on: [self-hosted, linux, x64] + strategy: + matrix: + config: + - 'debug' + - 'release' + arch: + - 'x86_64' + env: + VCPKG_VERSION: 2022.03.10 + + steps: + - uses: actions/checkout@v2 + + - name: Set up GCC 11 + uses: egor-tensin/setup-gcc@v1 + with: + version: 11 + platform: x64 + + - name: Setup xmake + uses: xmake-io/github-action-setup-xmake@v1 + with: + xmake-version: 2.6.4 + + - name: Installing vcpkg + run: | + git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg + cd .vcpkg + ./bootstrap-vcpkg.sh + cd .. + + - name: Cache vcpkg + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/.vcpkg/installed + key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + + - name: Config + run: | + xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: Build & Test + run: | + xmake test -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -- cgit v1.2.3 From 49ad6da1f77daa8c91a087046d82ab78d2e41314 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 12:34:51 +0200 Subject: If a bucket is in m_BucketMap it is OK, no need for separate flag --- zenserver/cache/structuredcachestore.cpp | 77 +++++++++----------------------- zenserver/cache/structuredcachestore.h | 4 +- 2 files changed, 22 insertions(+), 59 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index f3f6503f3..bb85f9824 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -572,7 +572,7 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue ////////////////////////////////////////////////////////////////////////// -ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)) +ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)), m_BucketId(Oid::Zero) { } @@ -580,7 +580,7 @@ ZenCacheDiskLayer::CacheBucket::~CacheBucket() { } -void +bool ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate) { using namespace std::literals; @@ -598,7 +598,10 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo if (Manifest) { m_BucketId = Manifest["BucketId"].AsObjectId(); - m_IsOk = m_BucketId != Oid::Zero; + if (m_BucketId == Oid::Zero) + { + return false; + } } else if (AllowCreate) { @@ -612,7 +615,7 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo } else { - return; + return false; } OpenLog(BucketDir, IsNew); @@ -628,7 +631,7 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo } } - m_IsOk = true; + return true; } void @@ -1208,7 +1211,6 @@ void ZenCacheDiskLayer::CacheBucket::Drop() { RwLock::ExclusiveLockScope _(m_IndexLock); - m_IsOk = false; std::vector> ShardLocks; ShardLocks.reserve(256); @@ -1724,10 +1726,6 @@ ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx) for (auto& Kv : m_Buckets) { CacheBucket& Bucket = *Kv.second; - if (!Bucket.IsOk()) - { - continue; - } Bucket.CollectGarbage(GcCtx); } } @@ -1742,10 +1740,6 @@ ZenCacheDiskLayer::UpdateAccessTimes(const zen::access_tracking::AccessTimes& Ac if (auto It = m_Buckets.find(Kv.first); It != m_Buckets.end()) { CacheBucket& Bucket = *It->second; - if (!Bucket.IsOk()) - { - continue; - } Bucket.UpdateAccessTimes(Kv.second); } } @@ -1959,10 +1953,6 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach if (it != m_Buckets.end()) { Bucket = it->second.get(); - if (!Bucket->IsOk()) - { - return false; - } } } @@ -1984,9 +1974,9 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach std::filesystem::path BucketPath = m_RootDir; BucketPath /= BucketName; - Bucket->OpenOrCreate(BucketPath); - if (!Bucket->IsOk()) + if (!Bucket->OpenOrCreate(BucketPath)) { + m_Buckets.erase(BucketName); return false; } } @@ -2010,10 +2000,6 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z if (it != m_Buckets.end()) { Bucket = it->second.get(); - if (!Bucket->IsOk()) - { - return; - } } } @@ -2035,9 +2021,9 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z std::filesystem::path BucketPath = m_RootDir; BucketPath /= BucketName; - Bucket->OpenOrCreate(BucketPath); - if (!Bucket->IsOk()) + if (!Bucket->OpenOrCreate(BucketPath)) { + m_Buckets.erase(BucketName); return; } } @@ -2064,25 +2050,20 @@ ZenCacheDiskLayer::DiscoverBuckets() // New bucket needs to be created if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) { + continue; } - else - { - auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); - CacheBucket& Bucket = *InsertResult.first->second; - Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false); + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique(BucketName)); + CacheBucket& Bucket = *InsertResult.first->second; - if (Bucket.IsOk()) - { - ZEN_INFO("Discovered bucket '{}'", BucketName); - } - else - { - ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); + if (!Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false)) + { + ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); - m_Buckets.erase(InsertResult.first); - } + m_Buckets.erase(InsertResult.first); + continue; } + ZEN_INFO("Discovered bucket '{}'", BucketName); } } @@ -2098,10 +2079,6 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) CacheBucket& Bucket = *it->second; m_DroppedBuckets.push_back(std::move(it->second)); m_Buckets.erase(it); - if (!Bucket.IsOk()) - { - return false; - } Bucket.Drop(); @@ -2125,10 +2102,6 @@ ZenCacheDiskLayer::Flush() for (auto& Kv : m_Buckets) { CacheBucket* Bucket = Kv.second.get(); - if (!Bucket->IsOk()) - { - continue; - } Buckets.push_back(Bucket); } } @@ -2147,10 +2120,6 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) for (auto& Kv : m_Buckets) { CacheBucket& Bucket = *Kv.second; - if (!Bucket.IsOk()) - { - continue; - } Bucket.Scrub(Ctx); } } @@ -2163,10 +2132,6 @@ ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx) for (auto& Kv : m_Buckets) { CacheBucket& Bucket = *Kv.second; - if (!Bucket.IsOk()) - { - continue; - } Bucket.GatherReferences(GcCtx); } } diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index bae15231b..311e062be 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -233,7 +233,7 @@ private: CacheBucket(std::string BucketName); ~CacheBucket(); - void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true); + bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true); bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); void Put(const IoHash& HashKey, const ZenCacheValue& Value); void Drop(); @@ -243,7 +243,6 @@ private: void CollectGarbage(GcContext& GcCtx); void UpdateAccessTimes(const std::vector& AccessTimes); - inline bool IsOk() const { return m_IsOk; } inline uint64_t TotalSize() const { return m_TotalSize.load(std::memory_order::relaxed); } private: @@ -255,7 +254,6 @@ private: std::filesystem::path m_BlocksBasePath; BlockStore m_BlockStore; Oid m_BucketId; - bool m_IsOk = false; uint64_t m_LargeObjectThreshold = 64 * 1024; // These files are used to manage storage of small objects for this bucket -- cgit v1.2.3 From 0e325ea21a56a154bdf4b92745095c588f69c817 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 12:39:59 +0200 Subject: dropIndex -> DropIndex --- zenserver/cache/structuredcachestore.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index bb85f9824..9ebec1e76 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1179,7 +1179,7 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& static bool DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view BucketName) { - int dropIndex = 0; + int DropIndex = 0; do { if (!std::filesystem::exists(BucketDir)) @@ -1187,11 +1187,11 @@ DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view Bu return false; } - std::string DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, dropIndex); + std::string DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, DropIndex); std::filesystem::path DroppedBucketPath = BucketDir.parent_path() / DroppedBucketName; if (std::filesystem::exists(DroppedBucketPath)) { - dropIndex++; + DropIndex++; continue; } -- cgit v1.2.3 From d0c46ee88f3b08c0abc4f1cfba40b966f3c6a59e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 13:49:58 +0200 Subject: Make sure ZenCacheMemoryLayer handles dropped buckets correctly (just like ZenCacheDiskLayer) --- zenserver/cache/structuredcachestore.cpp | 119 ++++++++++++++++++++++--------- zenserver/cache/structuredcachestore.h | 12 ++-- 2 files changed, 95 insertions(+), 36 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 9ebec1e76..d4b8bff39 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -404,14 +404,14 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa { RwLock::SharedLockScope _(m_Lock); - auto it = m_Buckets.find(std::string(InBucket)); + auto It = m_Buckets.find(std::string(InBucket)); - if (it == m_Buckets.end()) + if (It == m_Buckets.end()) { return false; } - CacheBucket* Bucket = &it->second; + CacheBucket* Bucket = It->second.get(); _.ReleaseNow(); @@ -425,14 +425,15 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa void ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value) { - CacheBucket* Bucket = nullptr; + const auto BucketName = std::string(InBucket); + CacheBucket* Bucket = nullptr; { RwLock::SharedLockScope _(m_Lock); if (auto It = m_Buckets.find(std::string(InBucket)); It != m_Buckets.end()) { - Bucket = &It->second; + Bucket = It->second.get(); } } @@ -444,11 +445,12 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const if (auto It = m_Buckets.find(std::string(InBucket)); It != m_Buckets.end()) { - Bucket = &It->second; + Bucket = It->second.get(); } else { - Bucket = &m_Buckets[std::string(InBucket)]; + auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique()); + Bucket = InsertResult.first->second.get(); } } @@ -458,11 +460,37 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const IoHash& HashKey, const } bool -ZenCacheMemoryLayer::DropBucket(std::string_view Bucket) +ZenCacheMemoryLayer::DropBucket(std::string_view InBucket) { RwLock::ExclusiveLockScope _(m_Lock); - return !!m_Buckets.erase(std::string(Bucket)); + auto It = m_Buckets.find(std::string(InBucket)); + + if (It != m_Buckets.end()) + { + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It); + Bucket.Drop(); + return true; + } + return false; +} + +void +ZenCacheMemoryLayer::Drop() +{ + RwLock::ExclusiveLockScope _(m_Lock); + std::vector> Buckets; + Buckets.reserve(m_Buckets.size()); + while (!m_Buckets.empty()) + { + const auto& It = m_Buckets.begin(); + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It->first); + Bucket.Drop(); + } } void @@ -472,7 +500,7 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx) for (auto& Kv : m_Buckets) { - Kv.second.Scrub(Ctx); + Kv.second->Scrub(Ctx); } } @@ -486,7 +514,7 @@ ZenCacheMemoryLayer::GatherAccessTimes(zen::access_tracking::AccessTimes& Access for (auto& Kv : m_Buckets) { std::vector& Bucket = AccessTimes.Buckets[Kv.first]; - Kv.second.GatherAccessTimes(Bucket); + Kv.second->GatherAccessTimes(Bucket); } } @@ -505,7 +533,7 @@ ZenCacheMemoryLayer::TotalSize() const for (auto& Kv : m_Buckets) { - TotalSize += Kv.second.TotalSize(); + TotalSize += Kv.second->TotalSize(); } return TotalSize; @@ -570,6 +598,13 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue m_TotalSize.fetch_add(Value.Value.GetSize(), std::memory_order::relaxed); } +void +ZenCacheMemoryLayer::CacheBucket::Drop() +{ + RwLock::ExclusiveLockScope _(m_BucketLock); + m_CacheMap.clear(); +} + ////////////////////////////////////////////////////////////////////////// ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)), m_BucketId(Oid::Zero) @@ -1207,7 +1242,7 @@ DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view Bu } while (true); } -void +bool ZenCacheDiskLayer::CacheBucket::Drop() { RwLock::ExclusiveLockScope _(m_IndexLock); @@ -1221,9 +1256,10 @@ ZenCacheDiskLayer::CacheBucket::Drop() m_BlockStore.Close(); m_SlogFile.Close(); - DeleteBucketFromDisk(m_BucketDir, m_BucketName); + bool Deleted = DeleteBucketFromDisk(m_BucketDir, m_BucketName); m_Index.clear(); + return Deleted; } void @@ -1948,11 +1984,11 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach { RwLock::SharedLockScope _(m_Lock); - auto it = m_Buckets.find(BucketName); + auto It = m_Buckets.find(BucketName); - if (it != m_Buckets.end()) + if (It != m_Buckets.end()) { - Bucket = it->second.get(); + Bucket = It->second.get(); } } @@ -1962,9 +1998,9 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach RwLock::ExclusiveLockScope _(m_Lock); - if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end()) + if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) { - Bucket = it->second.get(); + Bucket = It->second.get(); } else { @@ -1995,11 +2031,11 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z { RwLock::SharedLockScope _(m_Lock); - auto it = m_Buckets.find(BucketName); + auto It = m_Buckets.find(BucketName); - if (it != m_Buckets.end()) + if (It != m_Buckets.end()) { - Bucket = it->second.get(); + Bucket = It->second.get(); } } @@ -2009,9 +2045,9 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z RwLock::ExclusiveLockScope _(m_Lock); - if (auto it = m_Buckets.find(BucketName); it != m_Buckets.end()) + if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) { - Bucket = it->second.get(); + Bucket = It->second.get(); } else { @@ -2072,17 +2108,15 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) { RwLock::ExclusiveLockScope _(m_Lock); - auto it = m_Buckets.find(std::string(InBucket)); + auto It = m_Buckets.find(std::string(InBucket)); - if (it != m_Buckets.end()) + if (It != m_Buckets.end()) { - CacheBucket& Bucket = *it->second; - m_DroppedBuckets.push_back(std::move(it->second)); - m_Buckets.erase(it); + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It); - Bucket.Drop(); - - return true; + return Bucket.Drop(); } // Make sure we remove the folder even if we don't know about the bucket @@ -2091,6 +2125,27 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) return DeleteBucketFromDisk(BucketPath, InBucket); } +bool +ZenCacheDiskLayer::Drop() +{ + RwLock::ExclusiveLockScope _(m_Lock); + + std::vector> Buckets; + Buckets.reserve(m_Buckets.size()); + while (!m_Buckets.empty()) + { + const auto& It = m_Buckets.begin(); + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It->first); + if (!Bucket.Drop()) + { + return false; + } + } + return true; +} + void ZenCacheDiskLayer::Flush() { diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 311e062be..4da48d3d9 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -150,6 +150,7 @@ public: bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); + void Drop(); bool DropBucket(std::string_view Bucket); void Scrub(ScrubContext& Ctx); void GatherAccessTimes(zen::access_tracking::AccessTimes& AccessTimes); @@ -193,14 +194,16 @@ private: bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); void Put(const IoHash& HashKey, const ZenCacheValue& Value); + void Drop(); void Scrub(ScrubContext& Ctx); void GatherAccessTimes(std::vector& AccessTimes); inline uint64_t TotalSize() const { return m_TotalSize; } }; - mutable RwLock m_Lock; - std::unordered_map m_Buckets; - Configuration m_Configuration; + mutable RwLock m_Lock; + std::unordered_map> m_Buckets; + std::vector> m_DroppedBuckets; + Configuration m_Configuration; ZenCacheMemoryLayer(const ZenCacheMemoryLayer&) = delete; ZenCacheMemoryLayer& operator=(const ZenCacheMemoryLayer&) = delete; @@ -214,6 +217,7 @@ public: bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); + bool Drop(); bool DropBucket(std::string_view Bucket); void Flush(); void Scrub(ScrubContext& Ctx); @@ -236,7 +240,7 @@ private: bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true); bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); void Put(const IoHash& HashKey, const ZenCacheValue& Value); - void Drop(); + bool Drop(); void Flush(); void Scrub(ScrubContext& Ctx); void GatherReferences(GcContext& GcCtx); -- cgit v1.2.3 From 5feb2725a3c6461e913a95ea271046855c773ac5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 14:08:11 +0200 Subject: namespace drop --- zenserver/cache/structuredcache.cpp | 18 +++++++++--------- zenserver/cache/structuredcachestore.cpp | 25 ++++++++++++++++++++++++- zenserver/cache/structuredcachestore.h | 7 +++++-- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index e11499289..79d15a204 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -398,7 +398,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest& Request, std::string_view) +HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest& Request, std::string_view Namespace) { switch (Request.RequestVerb()) { @@ -412,14 +412,14 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(zen::HttpServerRequest& case HttpVerb::kDelete: // Drop namespace { - // if (m_CacheStore.DropNamespace(Namespace)) - // { - // return Request.WriteResponse(HttpResponseCode::OK); - // } - // else - // { - // return Request.WriteResponse(HttpResponseCode::NotFound); - // } + if (m_CacheStore.DropNamespace(Namespace)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } } break; diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index d4b8bff39..6af673ebc 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -342,6 +342,14 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket) return AnyDropped; } +bool +ZenCacheNamespace::Drop() +{ + m_MemLayer.Drop(); + const bool DiskDropped = m_DiskLayer.Drop(); + return DiskDropped; +} + void ZenCacheNamespace::Flush() { @@ -2300,7 +2308,22 @@ ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket) { return Store->DropBucket(Bucket); } - ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put, bucket '{}'", Namespace, Bucket); + ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropBucket, bucket '{}'", Namespace, Bucket); + return false; +} + +bool +ZenCacheStore::DropNamespace(std::string_view InNamespace) +{ + RwLock::SharedLockScope _(m_NamespacesLock); + if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end()) + { + ZenCacheNamespace& Namespace = *It->second; + m_DroppedNamespaces.push_back(std::move(It->second)); + m_Namespaces.erase(It); + return Namespace.Drop(); + } + ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace); return false; } diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 4da48d3d9..c07c3e382 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -333,6 +333,7 @@ public: bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); + bool Drop(); bool DropBucket(std::string_view Bucket); void Flush(); void Scrub(ScrubContext& Ctx); @@ -375,6 +376,7 @@ public: bool Get(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); void Put(std::string_view Namespace, std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value); bool DropBucket(std::string_view Namespace, std::string_view Bucket); + bool DropNamespace(std::string_view Namespace); void Flush(); void Scrub(ScrubContext& Ctx); @@ -388,8 +390,9 @@ private: typedef std::unordered_map> NameSpaceMap; - mutable RwLock m_NamespacesLock; - NameSpaceMap m_Namespaces; + mutable RwLock m_NamespacesLock; + NameSpaceMap m_Namespaces; + std::vector> m_DroppedNamespaces; CasGc& m_Gc; Configuration m_Configuration; -- cgit v1.2.3 From de5ea33889ee2ccf09042360007d6ac54e502a9b Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 25 May 2022 14:09:27 +0200 Subject: Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3fc465dd..0c418324f 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ is incredibly handy. When that is installed you may enable auto-attach to child * `zencore-test` exercises unit tests in the zencore project * `zenserver-test` exercises the zen server itself (functional tests) -The tests are implemented using [doctest](https://github.com/onqtam/doctest), which is similar to Catch in usage. +The tests are implemented using [doctest](https://github.com/onqtam/doctest), which is similar to Catch in usage. We now also support [catch2](https://github.com/catchorg/Catch2) # Adding a http.sys URL reservation (Windows only) -- cgit v1.2.3 From 1a3709fd35f592b146d794835ab0e1a95ee078e0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 14:17:59 +0200 Subject: clean up namespace folders --- zenserver/cache/structuredcachestore.cpp | 78 +++++++++++++++++--------------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 6af673ebc..aa3be3974 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -205,6 +205,36 @@ namespace { return true; } + bool MoveAndDeleteDirectory(const std::filesystem::path& Dir) + { + int DropIndex = 0; + do + { + if (!std::filesystem::exists(Dir)) + { + return false; + } + + std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex); + std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName; + if (std::filesystem::exists(DroppedBucketPath)) + { + DropIndex++; + continue; + } + + std::error_code Ec; + std::filesystem::rename(Dir, DroppedBucketPath, Ec); + if (!Ec) + { + DeleteDirectories(DroppedBucketPath); + return true; + } + // TODO: Do we need to bail at some point? + zen::Sleep(100); + } while (true); + } + } // namespace namespace fs = std::filesystem; @@ -346,8 +376,11 @@ bool ZenCacheNamespace::Drop() { m_MemLayer.Drop(); - const bool DiskDropped = m_DiskLayer.Drop(); - return DiskDropped; + if (!m_DiskLayer.Drop()) + { + return false; + } + return MoveAndDeleteDirectory(m_RootDir); } void @@ -1219,37 +1252,6 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& PutInlineCacheValue(HashKey, Value); } -static bool -DeleteBucketFromDisk(const std::filesystem::path& BucketDir, std::string_view BucketName) -{ - int DropIndex = 0; - do - { - if (!std::filesystem::exists(BucketDir)) - { - return false; - } - - std::string DroppedBucketName = fmt::format("[dropped]{}({})", BucketName, DropIndex); - std::filesystem::path DroppedBucketPath = BucketDir.parent_path() / DroppedBucketName; - if (std::filesystem::exists(DroppedBucketPath)) - { - DropIndex++; - continue; - } - - std::error_code Ec; - std::filesystem::rename(BucketDir, DroppedBucketPath, Ec); - if (!Ec) - { - DeleteDirectories(DroppedBucketPath); - return true; - } - // TODO: Do we need to bail at some point? - zen::Sleep(100); - } while (true); -} - bool ZenCacheDiskLayer::CacheBucket::Drop() { @@ -1264,7 +1266,7 @@ ZenCacheDiskLayer::CacheBucket::Drop() m_BlockStore.Close(); m_SlogFile.Close(); - bool Deleted = DeleteBucketFromDisk(m_BucketDir, m_BucketName); + bool Deleted = MoveAndDeleteDirectory(m_BucketDir); m_Index.clear(); return Deleted; @@ -2130,7 +2132,7 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) // Make sure we remove the folder even if we don't know about the bucket std::filesystem::path BucketPath = m_RootDir; BucketPath /= std::string(InBucket); - return DeleteBucketFromDisk(BucketPath, InBucket); + return MoveAndDeleteDirectory(BucketPath); } bool @@ -2150,6 +2152,7 @@ ZenCacheDiskLayer::Drop() { return false; } + return MoveAndDeleteDirectory(m_RootDir); } return true; } @@ -2321,7 +2324,10 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace) ZenCacheNamespace& Namespace = *It->second; m_DroppedNamespaces.push_back(std::move(It->second)); m_Namespaces.erase(It); - return Namespace.Drop(); + if (!Namespace.Drop()) + { + return false; + } } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace); return false; -- cgit v1.2.3 From 91283d188c6b954874d888714d54071414364b53 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 14:39:40 +0200 Subject: bugfixes and test for namespace drop --- zenserver/cache/structuredcachestore.cpp | 107 ++++++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 15 deletions(-) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index aa3be3974..e4dbf390a 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -376,11 +376,7 @@ bool ZenCacheNamespace::Drop() { m_MemLayer.Drop(); - if (!m_DiskLayer.Drop()) - { - return false; - } - return MoveAndDeleteDirectory(m_RootDir); + return m_DiskLayer.Drop(); } void @@ -2152,9 +2148,8 @@ ZenCacheDiskLayer::Drop() { return false; } - return MoveAndDeleteDirectory(m_RootDir); } - return true; + return MoveAndDeleteDirectory(m_RootDir); } void @@ -2324,10 +2319,7 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace) ZenCacheNamespace& Namespace = *It->second; m_DroppedNamespaces.push_back(std::move(It->second)); m_Namespaces.erase(It); - if (!Namespace.Drop()) - { - return false; - } + return Namespace.Drop(); } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace); return false; @@ -3280,25 +3272,110 @@ TEST_CASE("z$.drop.bucket") Workers.ScheduleWork([&]() { zen::Sleep(100); Value1.Value = IoBuffer{}; - Value1 = GetValue(Zcs, Namespace, Bucket, Key1); - CHECK(!Value1.Value); WorkComplete = true; }); // On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket // Our DropBucket execution blocks any incoming request from completing until we are done with the drop - Zcs.DropBucket(Namespace, Bucket); + CHECK(Zcs.DropBucket(Namespace, Bucket)); while (!WorkComplete) { zen::Sleep(1); } - CHECK(!Value1.Value); // Entire bucket should be dropped, but doing a request should will re-create the namespace but it must still be empty + Value1 = GetValue(Zcs, Namespace, Bucket, Key1); + CHECK(!Value1.Value); ZenCacheValue Value2 = GetValue(Zcs, Namespace, Bucket, Key2); CHECK(!Value2.Value); } } +TEST_CASE("z$.drop.namespace") +{ + using namespace testutils; + + const auto CreateCacheValue = [](size_t Size) -> CbObject { + std::vector Buf; + Buf.resize(Size); + + CbObjectWriter Writer; + Writer.AddBinary("Binary"sv, Buf.data(), Buf.size()); + return Writer.Save(); + }; + + ScopedTemporaryDirectory TempDir; + CreateDirectories(TempDir.Path()); + + auto PutValue = + [&CreateCacheValue](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, size_t KeyIndex, size_t Size) { + // Create a cache record + IoHash Key = CreateKey(KeyIndex); + CbObject CacheValue = CreateCacheValue(Size); + + IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); + Buffer.SetContentType(ZenContentType::kCbObject); + + ZenCacheValue PutValue = {.Value = Buffer}; + Zcs.Put(Namespace, Bucket, Key, PutValue); + return Key; + }; + auto GetValue = [](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) { + ZenCacheValue GetValue; + Zcs.Get(Namespace, Bucket, Key, GetValue); + return GetValue; + }; + WorkerThreadPool Workers(1); + { + CasGc Gc; + ZenCacheStore Zcs(Gc, {.BasePath = TempDir.Path() / "cache", .AllowAutomaticCreationOfNamespaces = true}); + const auto Bucket1 = "teardrinker1"sv; + const auto Bucket2 = "teardrinker2"sv; + const auto Namespace1 = "mynamespace1"sv; + const auto Namespace2 = "mynamespace2"sv; + + IoHash Key1 = PutValue(Zcs, Namespace1, Bucket1, 42, 4096); + IoHash Key2 = PutValue(Zcs, Namespace1, Bucket2, 43, 2048); + IoHash Key3 = PutValue(Zcs, Namespace2, Bucket1, 44, 4096); + IoHash Key4 = PutValue(Zcs, Namespace2, Bucket2, 45, 2048); + + ZenCacheValue Value1 = GetValue(Zcs, Namespace1, Bucket1, Key1); + CHECK(Value1.Value); + ZenCacheValue Value2 = GetValue(Zcs, Namespace1, Bucket2, Key2); + CHECK(Value2.Value); + ZenCacheValue Value3 = GetValue(Zcs, Namespace2, Bucket1, Key3); + CHECK(Value3.Value); + ZenCacheValue Value4 = GetValue(Zcs, Namespace2, Bucket2, Key4); + CHECK(Value4.Value); + + std::atomic_bool WorkComplete = false; + Workers.ScheduleWork([&]() { + zen::Sleep(100); + Value1.Value = IoBuffer{}; + Value2.Value = IoBuffer{}; + Value3.Value = IoBuffer{}; + Value4.Value = IoBuffer{}; + WorkComplete = true; + }); + // On Windows, DropBucket() will be blocked as long as we hold a reference to a buffer in the bucket + // Our DropBucket execution blocks any incoming request from completing until we are done with the drop + CHECK(Zcs.DropNamespace(Namespace1)); + while (!WorkComplete) + { + zen::Sleep(1); + } + + // Entire namespace should be dropped, but doing a request should will re-create the namespace but it must still be empty + Value1 = GetValue(Zcs, Namespace1, Bucket1, Key1); + CHECK(!Value1.Value); + Value2 = GetValue(Zcs, Namespace1, Bucket2, Key2); + CHECK(!Value2.Value); + Value3 = GetValue(Zcs, Namespace2, Bucket1, Key3); + CHECK(Value3.Value); + Value4 = GetValue(Zcs, Namespace2, Bucket2, Key4); + CHECK(Value4.Value); + } +} + TEST_CASE("z$.blocked.disklayer.put") { ScopedTemporaryDirectory TempDir; -- cgit v1.2.3 From d8564b9a02ebb89887e93a8951206faf82313607 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 15:33:40 +0200 Subject: NameSpaceMap -> NamespaceMap --- zenserver/cache/structuredcachestore.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index c07c3e382..c226074fe 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -388,10 +388,10 @@ private: ZenCacheNamespace* GetNamespace(std::string_view Namespace); void IterateNamespaces(const std::function& Callback) const; - typedef std::unordered_map> NameSpaceMap; + typedef std::unordered_map> NamespaceMap; mutable RwLock m_NamespacesLock; - NameSpaceMap m_Namespaces; + NamespaceMap m_Namespaces; std::vector> m_DroppedNamespaces; CasGc& m_Gc; -- cgit v1.2.3 From 682c7d68f8ba2d0bea16ef382c2b35a772b4c8bc Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 27 May 2022 08:25:50 +0200 Subject: Make sure we can properly create the block file before assigning it for use --- zenstore/blockstore.cpp | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp index 3e27b0d12..7d3d2f5bb 100644 --- a/zenstore/blockstore.cpp +++ b/zenstore/blockstore.cpp @@ -225,23 +225,27 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, Writ { m_WriteBlock = nullptr; } + + if (m_ChunkBlocks.size() == m_MaxBlockCount) { - if (m_ChunkBlocks.size() == m_MaxBlockCount) - { - throw std::runtime_error(fmt::format("unable to allocate a new block in '{}'", m_BlocksBasePath)); - } - WriteBlockIndex += IsWriting ? 1 : 0; - while (m_ChunkBlocks.contains(WriteBlockIndex)) - { - WriteBlockIndex = (WriteBlockIndex + 1) & (m_MaxBlockCount - 1); - } - std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, WriteBlockIndex); - m_WriteBlock = new BlockStoreFile(BlockPath); - m_ChunkBlocks[WriteBlockIndex] = m_WriteBlock; - m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release); + throw std::runtime_error(fmt::format("unable to allocate a new block in '{}'", m_BlocksBasePath)); + } + + WriteBlockIndex += IsWriting ? 1 : 0; + while (m_ChunkBlocks.contains(WriteBlockIndex)) + { + WriteBlockIndex = (WriteBlockIndex + 1) & (m_MaxBlockCount - 1); } + + std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, WriteBlockIndex); + + Ref NewBlockFile = new BlockStoreFile(BlockPath); + NewBlockFile->Create(m_MaxBlockSize); + + m_ChunkBlocks[WriteBlockIndex] = NewBlockFile; + m_WriteBlock = NewBlockFile; + m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release); m_CurrentInsertOffset = 0; - m_WriteBlock->Create(m_MaxBlockSize); } uint64_t InsertOffset = m_CurrentInsertOffset; m_CurrentInsertOffset = RoundUp(InsertOffset + Size, Alignment); -- cgit v1.2.3 From 7f790cc5e68d498b3d857a13e40a2c62fca87a6c Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Fri, 27 May 2022 10:38:36 -0700 Subject: Horde execute compressed input blobs (#109) --- zenserver/upstream/hordecompute.cpp | 163 +++++++++++++++++++++++++++--------- 1 file changed, 123 insertions(+), 40 deletions(-) diff --git a/zenserver/upstream/hordecompute.cpp b/zenserver/upstream/hordecompute.cpp index 2ec24b303..38c798569 100644 --- a/zenserver/upstream/hordecompute.cpp +++ b/zenserver/upstream/hordecompute.cpp @@ -157,8 +157,13 @@ namespace detail { { ApplyResult.Timepoints["zen-storage-build-ref"] = DateTime::NowTicks(); - std::scoped_lock Lock(m_TaskMutex); - if (m_PendingTasks.contains(UpstreamData.TaskId)) + + bool AlreadyQueued; + { + std::scoped_lock Lock(m_TaskMutex); + AlreadyQueued = m_PendingTasks.contains(UpstreamData.TaskId); + } + if (AlreadyQueued) { // Pending task is already queued, return success ApplyResult.Success = true; @@ -171,7 +176,7 @@ namespace detail { CloudCacheSession StorageSession(m_StorageClient); { - CloudCacheResult Result = BatchPutBlobsIfMissing(StorageSession, UpstreamData.Blobs); + CloudCacheResult Result = BatchPutBlobsIfMissing(StorageSession, UpstreamData.Blobs, UpstreamData.CasIds); ApplyResult.Bytes += Result.Bytes; ApplyResult.ElapsedSeconds += Result.ElapsedSeconds; ApplyResult.Timepoints["zen-storage-upload-blobs"] = DateTime::NowTicks(); @@ -182,6 +187,22 @@ namespace detail { return ApplyResult; } UpstreamData.Blobs.clear(); + UpstreamData.CasIds.clear(); + } + + { + CloudCacheResult Result = BatchPutCompressedBlobsIfMissing(StorageSession, UpstreamData.Cids); + ApplyResult.Bytes += Result.Bytes; + ApplyResult.ElapsedSeconds += Result.ElapsedSeconds; + ApplyResult.Timepoints["zen-storage-upload-compressed-blobs"] = DateTime::NowTicks(); + if (!Result.Success) + { + ApplyResult.Error = { + .ErrorCode = Result.ErrorCode, + .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to upload compressed blobs"}; + return ApplyResult; + } + UpstreamData.Cids.clear(); } { @@ -279,9 +300,11 @@ namespace detail { } } - [[nodiscard]] CloudCacheResult BatchPutBlobsIfMissing(CloudCacheSession& Session, const std::map& Blobs) + [[nodiscard]] CloudCacheResult BatchPutBlobsIfMissing(CloudCacheSession& Session, + const std::map& Blobs, + const std::set& CasIds) { - if (Blobs.size() == 0) + if (Blobs.size() == 0 && CasIds.size() == 0) { return {.Success = true}; } @@ -292,6 +315,7 @@ namespace detail { // Batch check for missing blobs std::set Keys; std::transform(Blobs.begin(), Blobs.end(), std::inserter(Keys, Keys.end()), [](const auto& It) { return It.first; }); + Keys.insert(CasIds.begin(), CasIds.end()); CloudCacheExistsResult ExistsResult = Session.BlobExists(Session.Client().DefaultBlobStoreNamespace(), Keys); Log().debug("Queried {} missing blobs Need={} Duration={}s Result={}", @@ -310,7 +334,22 @@ namespace detail { for (const auto& Hash : ExistsResult.Needs) { - CloudCacheResult Result = Session.PutBlob(Session.Client().DefaultBlobStoreNamespace(), Hash, Blobs.at(Hash)); + IoBuffer DataBuffer; + if (Blobs.contains(Hash)) + { + DataBuffer = Blobs.at(Hash); + } + else + { + DataBuffer = m_CasStore.FindChunk(Hash); + if (!DataBuffer) + { + Log().warn("Put blob FAILED, input chunk '{}' missing", Hash); + return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .ErrorCode = -1, .Reason = "Failed to put blobs"}; + } + } + + CloudCacheResult Result = Session.PutBlob(Session.Client().DefaultBlobStoreNamespace(), Hash, DataBuffer); Log().debug("Put blob {} Bytes={} Duration={}s Result={}", Hash, Result.Bytes, Result.ElapsedSeconds, Result.Success); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; @@ -326,6 +365,62 @@ namespace detail { return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .Success = true}; } + [[nodiscard]] CloudCacheResult BatchPutCompressedBlobsIfMissing(CloudCacheSession& Session, const std::set& Cids) + { + if (Cids.size() == 0) + { + return {.Success = true}; + } + + int64_t Bytes{}; + double ElapsedSeconds{}; + + // Batch check for missing compressed blobs + CloudCacheExistsResult ExistsResult = Session.CompressedBlobExists(Session.Client().DefaultBlobStoreNamespace(), Cids); + Log().debug("Queried {} missing compressed blobs Need={} Duration={}s Result={}", + Cids.size(), + ExistsResult.Needs.size(), + ExistsResult.ElapsedSeconds, + ExistsResult.Success); + ElapsedSeconds += ExistsResult.ElapsedSeconds; + if (!ExistsResult.Success) + { + return { + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .ErrorCode = ExistsResult.ErrorCode ? ExistsResult.ErrorCode : -1, + .Reason = !ExistsResult.Reason.empty() ? std::move(ExistsResult.Reason) : "Failed to check if compressed blobs exist"}; + } + + for (const auto& Hash : ExistsResult.Needs) + { + IoBuffer DataBuffer = m_CidStore.FindChunkByCid(Hash); + if (!DataBuffer) + { + Log().warn("Put compressed blob FAILED, input CID chunk '{}' missing", Hash); + return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .ErrorCode = -1, .Reason = "Failed to put compressed blobs"}; + } + + CloudCacheResult Result = Session.PutCompressedBlob(Session.Client().DefaultBlobStoreNamespace(), Hash, DataBuffer); + Log().debug("Put compressed blob {} Bytes={} Duration={}s Result={}", + Hash, + Result.Bytes, + Result.ElapsedSeconds, + Result.Success); + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) + { + return {.Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1, + .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to put compressed blobs"}; + } + } + + return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .Success = true}; + } + [[nodiscard]] CloudCacheResult BatchPutObjectsIfMissing(CloudCacheSession& Session, const std::map& Objects) { if (Objects.size() == 0) @@ -599,6 +694,8 @@ namespace detail { { std::map Blobs; std::map Objects; + std::set CasIds; + std::set Cids; IoHash TaskId; IoHash RequirementsId; }; @@ -957,7 +1054,7 @@ namespace detail { for (auto& It : ApplyRecord.WorkerDescriptor["executables"sv]) { CbObjectView FileEntry = It.AsObjectView(); - if (!ProcessFileEntry(FileEntry, InputFiles, InputFileHashes, Data.Blobs)) + if (!ProcessFileEntry(FileEntry, InputFiles, InputFileHashes, Data.CasIds)) { return false; } @@ -966,7 +1063,7 @@ namespace detail { for (auto& It : ApplyRecord.WorkerDescriptor["files"sv]) { CbObjectView FileEntry = It.AsObjectView(); - if (!ProcessFileEntry(FileEntry, InputFiles, InputFileHashes, Data.Blobs)) + if (!ProcessFileEntry(FileEntry, InputFiles, InputFileHashes, Data.CasIds)) { return false; } @@ -1034,11 +1131,10 @@ namespace detail { bool AnyErrors = false; ApplyRecord.Action.IterateAttachments([&](CbFieldView Field) { - const IoHash Cid = Field.AsHash(); - const std::filesystem::path FilePath = {InputPath / Cid.ToHexString()}; - IoBuffer DataBuffer = m_CidStore.FindChunkByCid(Cid); + const IoHash Cid = Field.AsHash(); + const std::filesystem::path FilePath = {InputPath / Cid.ToHexString()}; - if (!DataBuffer) + if (!m_CidStore.ContainsChunk(Cid)) { Log().warn("process apply upstream FAILED, input CID chunk '{}' missing", Cid); AnyErrors = true; @@ -1050,11 +1146,9 @@ namespace detail { return; } - const IoHash CompressedId = IoHash::HashBuffer(DataBuffer.GetData(), DataBuffer.GetSize()); - InputFiles.insert(FilePath); - InputFileHashes[FilePath] = CompressedId; - Data.Blobs[CompressedId] = std::move(DataBuffer); + InputFileHashes[FilePath] = Cid; + Data.Cids.insert(Cid); }); if (AnyErrors) @@ -1067,7 +1161,7 @@ namespace detail { const UpstreamDirectory RootDirectory = BuildDirectoryTree(InputFiles); - CbObject Sandbox = BuildMerkleTreeDirectory(RootDirectory, InputFileHashes, Data.Blobs, Data.Objects); + CbObject Sandbox = BuildMerkleTreeDirectory(RootDirectory, InputFileHashes, Data.Cids, Data.Objects); const IoHash SandboxHash = Sandbox.GetHash(); Data.Objects[SandboxHash] = std::move(Sandbox); @@ -1118,28 +1212,18 @@ namespace detail { [[nodiscard]] bool ProcessFileEntry(const CbObjectView& FileEntry, std::set& InputFiles, std::map& InputFileHashes, - std::map& Blobs) + std::set& CasIds) { - const std::filesystem::path FilePath = FileEntry["name"sv].AsString(); - const IoHash ChunkId = FileEntry["hash"sv].AsHash(); - const uint64_t Size = FileEntry["size"sv].AsUInt64(); - IoBuffer DataBuffer = m_CasStore.FindChunk(ChunkId); + const std::filesystem::path FilePath = FileEntry["name"sv].AsString(); + const IoHash ChunkId = FileEntry["hash"sv].AsHash(); + const uint64_t Size = FileEntry["size"sv].AsUInt64(); - if (!DataBuffer) + if (!m_CasStore.ContainsChunk(ChunkId)) { Log().warn("process apply upstream FAILED, worker CAS chunk '{}' missing", ChunkId); return false; } - if (DataBuffer.Size() != Size) - { - Log().warn("process apply upstream FAILED, worker CAS chunk '{}' size: {}, action spec expected {}", - ChunkId, - DataBuffer.Size(), - Size); - return false; - } - if (InputFiles.contains(FilePath)) { Log().warn("process apply upstream FAILED, worker CAS chunk '{}' size: {} duplicate filename {}", ChunkId, Size, FilePath); @@ -1148,7 +1232,7 @@ namespace detail { InputFiles.insert(FilePath); InputFileHashes[FilePath] = ChunkId; - Blobs[ChunkId] = std::move(DataBuffer); + CasIds.insert(ChunkId); return true; } @@ -1204,7 +1288,7 @@ namespace detail { [[nodiscard]] CbObject BuildMerkleTreeDirectory(const UpstreamDirectory& RootDirectory, const std::map& InputFileHashes, - const std::map& Blobs, + const std::set& Cids, std::map& Objects) { CbObjectWriter DirectoryTreeWriter; @@ -1214,14 +1298,13 @@ namespace detail { DirectoryTreeWriter.BeginArray("f"sv); for (const auto& File : RootDirectory.Files) { - const std::filesystem::path FilePath = {RootDirectory.Path / File}; - const IoHash& FileHash = InputFileHashes.at(FilePath); - const uint64_t FileSize = Blobs.at(FileHash).Size(); + const std::filesystem::path FilePath = {RootDirectory.Path / File}; + const IoHash& FileHash = InputFileHashes.at(FilePath); + const bool Compressed = Cids.contains(FileHash); DirectoryTreeWriter.BeginObject(); DirectoryTreeWriter.AddString("n"sv, File); DirectoryTreeWriter.AddBinaryAttachment("h"sv, FileHash); - DirectoryTreeWriter.AddInteger("s"sv, FileSize); // Size - // DirectoryTreeWriter.AddInteger("a"sv, 0); // Attributes Currently unneeded + DirectoryTreeWriter.AddBool("c"sv, Compressed); DirectoryTreeWriter.EndObject(); } DirectoryTreeWriter.EndArray(); @@ -1232,7 +1315,7 @@ namespace detail { DirectoryTreeWriter.BeginArray("d"sv); for (const auto& Item : RootDirectory.Directories) { - CbObject Directory = BuildMerkleTreeDirectory(Item.second, InputFileHashes, Blobs, Objects); + CbObject Directory = BuildMerkleTreeDirectory(Item.second, InputFileHashes, Cids, Objects); const IoHash DirectoryHash = Directory.GetHash(); Objects[DirectoryHash] = std::move(Directory); -- cgit v1.2.3 From da0ca23fa55f7e900853bdfd06523b78fce32259 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 28 May 2022 00:29:17 +0200 Subject: Enable FILE_SHARE_DELETE on standalone files in disk buckets This allows us to delete the file even if it is open for read. We do a delete, the rename since we are not allowed to do a rename-overwrite, only delete. As we have the shard lock for the file we want to replace we can safely do a delete+rename. In the rare case that we fail to rename the file into place the old data is lost. As this is a *cache* and it should be very rare this is OK. --- zencore/include/zencore/iobuffer.h | 1 + zencore/iobuffer.cpp | 45 ++++++++ zenserver/cache/structuredcachestore.cpp | 184 +++++++++++++------------------ 3 files changed, 123 insertions(+), 107 deletions(-) diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 5d9daa1c7..bf658922d 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -403,6 +403,7 @@ class IoBufferBuilder { public: ZENCORE_API static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromFileWithSharedDelete(const std::filesystem::path& FileName); ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); ZENCORE_API static IoBuffer ReadFromFileMaybe(IoBuffer& InBuffer); diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index c4b7f7bdf..61db25d0f 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -469,6 +469,51 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); } +IoBuffer +IoBufferBuilder::MakeFromFileWithSharedDelete(const std::filesystem::path& FileName) +{ + uint64_t Size; + +#if ZEN_PLATFORM_WINDOWS + CAtlFile DataFile; + + HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING); + + if (FAILED(hRes)) + { + return {}; + } + + DataFile.GetSize((ULONGLONG&)Size); +#else + int Fd = open(FileName.c_str(), O_RDONLY | O_CLOEXEC); + if (Fd < 0) + { + return {}; + } + + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); + Size = Stat.st_size; +#endif // ZEN_PLATFORM_WINDOWS + + if (Size) + { +#if ZEN_PLATFORM_WINDOWS + void* Fd = DataFile.Detach(); +#endif + return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), 0, Size); + } + +#if !ZEN_PLATFORM_WINDOWS + close(Fd); +#endif + + // For an empty file, we may as well just return an empty memory IoBuffer + return IoBuffer(IoBuffer::Wrap, "", 0); +} + IoBuffer IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) { diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index e4dbf390a..d1b242c5c 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1206,7 +1206,7 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); - if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.ToPath())) + if (IoBuffer Data = IoBufferBuilder::MakeFromFileWithSharedDelete(DataFilePath.ToPath())) { OutValue.Value = Data; OutValue.Value.SetContentType(Loc.GetContentType()); @@ -1831,110 +1831,79 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c BuildPath(DataFilePath, HashKey); std::filesystem::path FsPath{DataFilePath.ToPath()}; - // We retry to move the file since it can be held open for read. - // This happens if the server processes a Get request for the file or - // if we are busy sending the file upstream - int RetryCount = 4; - do - { - Ec.clear(); - { - RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); - - DataFile.MoveTemporaryIntoPlace(FsPath, Ec); - - // Once we have called MoveTemporaryIntoPlace automatic clean up the temp file - // will be disabled as the file handle has already been closed - CleanUpTempFile = Ec ? true : false; + RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); - if (Ec) - { - std::error_code ExistingEc; - uint64_t OldFileSize = std::filesystem::file_size(FsPath, ExistingEc); - if (!ExistingEc && (OldFileSize == NewFileSize)) - { - ZEN_INFO( - "Failed to move temporary file '{}' to '{}' for '{}'. Target file has same size, assuming concurrent write of same " - "value, " - "move " - "failed with reason '{}'", - DataFile.GetPath(), - FsPath.string(), - m_BucketDir, - Ec.message()); - return; - } - } - } + std::filesystem::remove(FsPath, Ec); + if (Ec && Ec.value() != ENOENT) + { + throw std::system_error(Ec, fmt::format("Failed to replace file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir)); + } - if (!Ec) + DataFile.MoveTemporaryIntoPlace(FsPath, Ec); + if (Ec) + { + std::filesystem::path ParentPath = FsPath.parent_path(); + if (std::filesystem::is_directory(ParentPath)) { - uint8_t EntryFlags = DiskLocation::kStandaloneFile; - - if (Value.Value.GetContentType() == ZenContentType::kCbObject) - { - EntryFlags |= DiskLocation::kStructured; - } - else if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary) - { - EntryFlags |= DiskLocation::kCompressed; - } - - DiskLocation Loc(NewFileSize, EntryFlags); - IndexEntry Entry = IndexEntry(Loc, GcClock::TickCount()); - - uint64_t OldFileSize = 0; - RwLock::ExclusiveLockScope _(m_IndexLock); - if (auto It = m_Index.find(HashKey); It == m_Index.end()) - { - // Previously unknown object - m_Index.insert({HashKey, Entry}); - } - else - { - // TODO: should check if write is idempotent and bail out if it is? - OldFileSize = It.value().Location.Size(); - It.value() = Entry; - } - - m_SlogFile.Append({.Key = HashKey, .Location = Loc}); - if (OldFileSize <= NewFileSize) - { - m_TotalSize.fetch_add(NewFileSize - OldFileSize, std::memory_order::relaxed); - } - else - { - m_TotalSize.fetch_sub(OldFileSize - NewFileSize, std::memory_order::relaxed); - } - return; + throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir)); } - - std::filesystem::path ParentPath = FsPath.parent_path(); - if (!std::filesystem::is_directory(ParentPath)) + Ec.clear(); + std::filesystem::create_directories(ParentPath, Ec); + if (Ec) { - Ec.clear(); - std::filesystem::create_directories(ParentPath, Ec); - if (!Ec) - { - // Retry without sleep - continue; - } throw std::system_error( Ec, fmt::format("Failed to create parent directory '{}' for file '{}' for put in '{}'", ParentPath, FsPath, m_BucketDir)); } - ZEN_INFO("Failed renaming temporary file '{}' to '{}' for put in '{}', pausing and retrying, reason '{}'", - DataFile.GetPath().string(), - FsPath.string(), - m_BucketDir, - Ec.message()); + DataFile.MoveTemporaryIntoPlace(FsPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir)); + } + } + + // Once we have called MoveTemporaryIntoPlace automatic clean up the temp file + // will be disabled as the file handle has already been closed + CleanUpTempFile = false; - // Semi arbitrary back-off - zen::Sleep(200 * (5 - RetryCount)); // Sleep at most for a total of 3 seconds - } while (RetryCount-- > 0); + uint8_t EntryFlags = DiskLocation::kStandaloneFile; - throw std::system_error(Ec, fmt::format("Failed to finalize file '{}' for put in '{}'", DataFilePath.ToUtf8(), m_BucketDir)); + if (Value.Value.GetContentType() == ZenContentType::kCbObject) + { + EntryFlags |= DiskLocation::kStructured; + } + else if (Value.Value.GetContentType() == ZenContentType::kCompressedBinary) + { + EntryFlags |= DiskLocation::kCompressed; + } + + DiskLocation Loc(NewFileSize, EntryFlags); + IndexEntry Entry = IndexEntry(Loc, GcClock::TickCount()); + + uint64_t OldFileSize = 0; + RwLock::ExclusiveLockScope _(m_IndexLock); + if (auto It = m_Index.find(HashKey); It == m_Index.end()) + { + // Previously unknown object + m_Index.insert({HashKey, Entry}); + } + else + { + // TODO: should check if write is idempotent and bail out if it is? + OldFileSize = It.value().Location.Size(); + It.value() = Entry; + } + + m_SlogFile.Append({.Key = HashKey, .Location = Loc}); + if (OldFileSize <= NewFileSize) + { + m_TotalSize.fetch_add(NewFileSize - OldFileSize, std::memory_order::relaxed); + } + else + { + m_TotalSize.fetch_sub(OldFileSize - NewFileSize, std::memory_order::relaxed); + } } void @@ -3384,7 +3353,7 @@ TEST_CASE("z$.blocked.disklayer.put") const auto CreateCacheValue = [](size_t Size) -> CbObject { std::vector Buf; - Buf.resize(Size); + Buf.resize(Size, Size & 0xff); CbObjectWriter Writer; Writer.AddBinary("Binary"sv, Buf.data(), Buf.size()); @@ -3406,25 +3375,26 @@ TEST_CASE("z$.blocked.disklayer.put") ZenCacheValue BufferGet; CHECK(Zcs.Get("test_bucket", HashKey, BufferGet)); - // Overwriting with a value of same size should go fine - Zcs.Put("test_bucket", HashKey, {.Value = Buffer}); - CbObject CacheValue2 = CreateCacheValue(64 * 1024 + 64 + 1); IoBuffer Buffer2 = CacheValue2.GetBuffer().AsIoBuffer(); Buffer2.SetContentType(ZenContentType::kCbObject); -# if ZEN_PLATFORM_WINDOWS - // On Windows platform, overwriting with different size while we have - // it open for read should throw exception if file is held open - CHECK_THROWS(Zcs.Put("test_bucket", HashKey, {.Value = Buffer2})); -# else - // Other platforms should handle overwrite just fine + + // We should be able to overwrite even if the file is open for read Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}); -# endif - BufferGet = ZenCacheValue{}; + MemoryView OldView = BufferGet.Value.GetView(); - // Read access has been removed, we should now be able to overwrite it - Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}); + ZenCacheValue BufferGet2; + CHECK(Zcs.Get("test_bucket", HashKey, BufferGet2)); + MemoryView NewView = BufferGet2.Value.GetView(); + + // Make sure file openend for read before we wrote it still have old data + CHECK(OldView.GetSize() == Buffer.GetSize()); + CHECK(memcmp(OldView.GetData(), Buffer.GetData(), OldView.GetSize()) == 0); + + // Make sure we get the new data when reading after we write new data + CHECK(NewView.GetSize() == Buffer2.GetSize()); + CHECK(memcmp(NewView.GetData(), Buffer2.GetData(), NewView.GetSize()) == 0); } #endif -- cgit v1.2.3 From 13385d08ce740a57de3662beffaf5cca4586c7e5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 30 May 2022 13:59:48 +0200 Subject: add comment about removing files --- zenserver/cache/structuredcachestore.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index d1b242c5c..ee0835fd3 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -1833,6 +1833,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); + // We do a speculative remove of the file instead of probing with a exists call and check the error code instead std::filesystem::remove(FsPath, Ec); if (Ec && Ec.value() != ENOENT) { -- cgit v1.2.3 From 26fb535175e44d5819b385195971b5696e6d6122 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 30 May 2022 14:00:18 +0200 Subject: create local utility function for IoBufferBuilder::MakeFromFile* --- zencore/iobuffer.cpp | 73 ++++++++++++++++++---------------------------------- 1 file changed, 25 insertions(+), 48 deletions(-) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 61db25d0f..b948493a7 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -469,60 +469,20 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); } -IoBuffer -IoBufferBuilder::MakeFromFileWithSharedDelete(const std::filesystem::path& FileName) +static IoBuffer +MakeFromFileWithOptions(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size, bool UseShareDelete) { - uint64_t Size; + uint64_t FileSize; #if ZEN_PLATFORM_WINDOWS CAtlFile DataFile; - HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING); - - if (FAILED(hRes)) - { - return {}; - } - - DataFile.GetSize((ULONGLONG&)Size); -#else - int Fd = open(FileName.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd < 0) + DWORD ShareOptions = FILE_SHARE_READ; + if (UseShareDelete) { - return {}; + ShareOptions |= FILE_SHARE_DELETE; } - - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - fstat(Fd, &Stat); - Size = Stat.st_size; -#endif // ZEN_PLATFORM_WINDOWS - - if (Size) - { -#if ZEN_PLATFORM_WINDOWS - void* Fd = DataFile.Detach(); -#endif - return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), 0, Size); - } - -#if !ZEN_PLATFORM_WINDOWS - close(Fd); -#endif - - // For an empty file, we may as well just return an empty memory IoBuffer - return IoBuffer(IoBuffer::Wrap, "", 0); -} - -IoBuffer -IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) -{ - uint64_t FileSize; - -#if ZEN_PLATFORM_WINDOWS - CAtlFile DataFile; - - HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); + HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, FILE_SHARE_READ | ShareOptions, OPEN_EXISTING); if (FAILED(hRes)) { @@ -531,7 +491,12 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of DataFile.GetSize((ULONGLONG&)FileSize); #else - int Fd = open(FileName.c_str(), O_RDONLY); + int Flags = O_RDONLY; + if (UseShareDelete) + { + Flags |= O_CLOEXEC; + } + int Fd = open(FileName.c_str(), Flags); if (Fd < 0) { return {}; @@ -574,6 +539,18 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of return IoBuffer(IoBuffer::Wrap, "", 0); } +IoBuffer +IoBufferBuilder::MakeFromFileWithSharedDelete(const std::filesystem::path& FileName) +{ + return MakeFromFileWithOptions(FileName, ~0ull, 0, true); +} + +IoBuffer +IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) +{ + return MakeFromFileWithOptions(FileName, Offset, Size, false); +} + IoBuffer IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) { -- cgit v1.2.3 From 03325c9632b8bc603ef4e386a1113866bb51c5fe Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 30 May 2022 14:43:39 +0200 Subject: fix argument order in IoBufferBuilder::MakeFromFileWithSharedDelete --- zencore/iobuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index b948493a7..56b05d86d 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -542,7 +542,7 @@ MakeFromFileWithOptions(const std::filesystem::path& FileName, uint64_t Offset, IoBuffer IoBufferBuilder::MakeFromFileWithSharedDelete(const std::filesystem::path& FileName) { - return MakeFromFileWithOptions(FileName, ~0ull, 0, true); + return MakeFromFileWithOptions(FileName, 0, ~0ull, true); } IoBuffer -- cgit v1.2.3 From 8ff45dcce8013c5b2aba0c9c109354570fbf4b7a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 31 May 2022 00:01:47 +0200 Subject: Remove namespace from HTTP URI style request in upstream until shared instances are deployed with version that support si --- zenserver/upstream/zen.cpp | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index efc75b5b4..354233472 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -412,10 +412,11 @@ ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::strin { ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/"; - if (Namespace != ZenCacheStore::DefaultNamespace) - { - Uri << Namespace << "/"; - } + // TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support + // if (Namespace != ZenCacheStore::DefaultNamespace) + // { + // Uri << Namespace << "/"; + // } Uri << BucketId << "/" << Key.ToHexString(); cpr::Session& Session = m_SessionState->GetSession(); @@ -444,10 +445,11 @@ ZenStructuredCacheSession::GetCacheValue(std::string_view Namespace, { ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/"; - if (Namespace != ZenCacheStore::DefaultNamespace) - { - Uri << Namespace << "/"; - } + // TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support + // if (Namespace != ZenCacheStore::DefaultNamespace) + // { + // Uri << Namespace << "/"; + // } Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); cpr::Session& Session = m_SessionState->GetSession(); @@ -478,10 +480,11 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace, { ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/"; - if (Namespace != ZenCacheStore::DefaultNamespace) - { - Uri << Namespace << "/"; - } + // TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support + // if (Namespace != ZenCacheStore::DefaultNamespace) + // { + // Uri << Namespace << "/"; + // } Uri << BucketId << "/" << Key.ToHexString(); cpr::Session& Session = m_SessionState->GetSession(); @@ -515,10 +518,11 @@ ZenStructuredCacheSession::PutCacheValue(std::string_view Namespace, { ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/"; - if (Namespace != ZenCacheStore::DefaultNamespace) - { - Uri << Namespace << "/"; - } + // TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support + // if (Namespace != ZenCacheStore::DefaultNamespace) + // { + // Uri << Namespace << "/"; + // } Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); cpr::Session& Session = m_SessionState->GetSession(); -- cgit v1.2.3 From 350f62fff551988783d27425cddf1c1bec318c7e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 31 May 2022 00:04:21 +0200 Subject: remove unused parameter --- zenserver/upstream/zen.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 354233472..8fc1503c7 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -408,7 +408,7 @@ ZenStructuredCacheSession::CheckHealth() } ZenCacheResult -ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type) +ZenStructuredCacheSession::GetCacheRecord(std::string_view, std::string_view BucketId, const IoHash& Key, ZenContentType Type) { ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/"; @@ -438,10 +438,7 @@ ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::strin } ZenCacheResult -ZenStructuredCacheSession::GetCacheValue(std::string_view Namespace, - std::string_view BucketId, - const IoHash& Key, - const IoHash& ValueContentId) +ZenStructuredCacheSession::GetCacheValue(std::string_view, std::string_view BucketId, const IoHash& Key, const IoHash& ValueContentId) { ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/"; @@ -472,7 +469,7 @@ ZenStructuredCacheSession::GetCacheValue(std::string_view Namespace, } ZenCacheResult -ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace, +ZenStructuredCacheSession::PutCacheRecord(std::string_view, std::string_view BucketId, const IoHash& Key, IoBuffer Value, @@ -510,7 +507,7 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace, } ZenCacheResult -ZenStructuredCacheSession::PutCacheValue(std::string_view Namespace, +ZenStructuredCacheSession::PutCacheValue(std::string_view, std::string_view BucketId, const IoHash& Key, const IoHash& ValueContentId, -- cgit v1.2.3 From cc540692b75192229d0d8425c3fc0406784c1760 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 31 May 2022 22:04:34 +0200 Subject: Always block GC of current write block --- zenstore/blockstore.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp index 7d3d2f5bb..4e61c23cf 100644 --- a/zenstore/blockstore.cpp +++ b/zenstore/blockstore.cpp @@ -272,6 +272,10 @@ BlockStore::GetReclaimSnapshotState() { State.m_ActiveWriteBlocks.insert(BlockIndex); } + if (m_WriteBlock) + { + State.m_ActiveWriteBlocks.insert(m_WriteBlockIndex); + } State.BlockCount = m_ChunkBlocks.size(); return State; } -- cgit v1.2.3 From f8d110ad0ebf43bbae18db5bef5c38f3c37f5234 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 1 Jun 2022 10:23:43 +0200 Subject: Make it possible to configure GC monitoring interval command line: --gc-monitor-interval-seconds lua: monitorintervalseconds --- zenserver/config.cpp | 12 ++++++++++-- zenserver/config.h | 9 +++++---- zenserver/zenserver.cpp | 1 + 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/zenserver/config.cpp b/zenserver/config.cpp index be91ae4f8..0775eb736 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -478,6 +478,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "Size of gc disk reserve in bytes.", cxxopts::value(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), ""); + + options.add_option("gc", + "", + "gc-monitor-interval-seconds", + "Garbage collection interval in seconds. Default set to 0 (Off).", + cxxopts::value(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), + ""); try { auto result = options.parse(argc, argv); @@ -770,8 +777,9 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio if (sol::optional GcConfig = lua["gc"]) { - ServerOptions.GcConfig.IntervalSeconds = GcConfig.value().get_or("intervalseconds", 0); - ServerOptions.GcConfig.DiskReserveSize = GcConfig.value().get_or("diskreservesize", uint64_t(1u << 28)); + ServerOptions.GcConfig.MonitorIntervalSeconds = GcConfig.value().get_or("monitorintervalseconds", 30); + ServerOptions.GcConfig.IntervalSeconds = GcConfig.value().get_or("intervalseconds", 0); + ServerOptions.GcConfig.DiskReserveSize = GcConfig.value().get_or("diskreservesize", uint64_t(1u << 28)); if (sol::optional CacheGcConfig = GcConfig.value()["cache"]) { diff --git a/zenserver/config.h b/zenserver/config.h index 49f039d8d..a07bba9a4 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -96,10 +96,11 @@ struct ZenGcConfig { ZenCasEvictionPolicy Cas; ZenCacheEvictionPolicy Cache; - int32_t IntervalSeconds = 0; - bool CollectSmallObjects = true; - bool Enabled = true; - uint64_t DiskReserveSize = 1ul << 28; + int32_t MonitorIntervalSeconds = 30; + int32_t IntervalSeconds = 0; + bool CollectSmallObjects = true; + bool Enabled = true; + uint64_t DiskReserveSize = 1ul << 28; }; struct ZenServerOptions diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index a924d9c81..4db69c265 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -364,6 +364,7 @@ public: ZEN_INFO("initializing GC, enabled '{}', interval {}s", ServerOptions.GcConfig.Enabled, ServerOptions.GcConfig.IntervalSeconds); zen::GcSchedulerConfig GcConfig{ .RootDirectory = m_DataRoot / "gc", + .MonitorInterval = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds), .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, -- cgit v1.2.3 From faa5ce722c1d2621bab7cc840da5a9bfe8d04d5d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 1 Jun 2022 10:52:22 +0200 Subject: option description --- zenserver/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 0775eb736..c534865dc 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -482,7 +482,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) options.add_option("gc", "", "gc-monitor-interval-seconds", - "Garbage collection interval in seconds. Default set to 0 (Off).", + "Garbage collection monitoring interval in seconds.", cxxopts::value(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), ""); try -- cgit v1.2.3 From f63824c2b2ca978780192587111024a8dfb0a7a2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 1 Jun 2022 23:31:14 +0200 Subject: keep "reason" from upstream response so we can present it even if the request fails without outright error --- zenserver/upstream/zen.cpp | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 8fc1503c7..0237ec346 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -465,7 +465,11 @@ ZenStructuredCacheSession::GetCacheValue(std::string_view, std::string_view Buck const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; + return {.Response = Buffer, + .Bytes = Response.downloaded_bytes, + .ElapsedSeconds = Response.elapsed, + .Reason = Response.reason, + .Success = Success}; } ZenCacheResult @@ -501,9 +505,8 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view, return {.ErrorCode = static_cast(Response.error.code), .Reason = std::move(Response.error.message)}; } - return {.Bytes = Response.uploaded_bytes, - .ElapsedSeconds = Response.elapsed, - .Success = (Response.status_code == 200 || Response.status_code == 201)}; + const bool Success = Response.status_code == 200 || Response.status_code == 201; + return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Reason = Response.reason, .Success = Success}; } ZenCacheResult @@ -536,9 +539,8 @@ ZenStructuredCacheSession::PutCacheValue(std::string_view, return {.ErrorCode = static_cast(Response.error.code), .Reason = std::move(Response.error.message)}; } - return {.Bytes = Response.uploaded_bytes, - .ElapsedSeconds = Response.elapsed, - .Success = (Response.status_code == 200 || Response.status_code == 201)}; + const bool Success = Response.status_code == 200 || Response.status_code == 201; + return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Reason = Response.reason, .Success = Success}; } ZenCacheResult @@ -567,7 +569,11 @@ ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = std::move(Buffer), .Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; + return {.Response = std::move(Buffer), + .Bytes = Response.uploaded_bytes, + .ElapsedSeconds = Response.elapsed, + .Reason = Response.reason, + .Success = Success}; } ZenCacheResult @@ -595,7 +601,11 @@ ZenStructuredCacheSession::InvokeRpc(const CbPackage& Request) const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = std::move(Buffer), .Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; + return {.Response = std::move(Buffer), + .Bytes = Response.uploaded_bytes, + .ElapsedSeconds = Response.elapsed, + .Reason = Response.reason, + .Success = Success}; } } // namespace zen -- cgit v1.2.3 From 0d739b955848725a77f818ac32db7b518fd7f878 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 2 Jun 2022 00:13:15 +0200 Subject: add timeout on pr validation --- .github/workflows/self_host_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/self_host_build.yml b/.github/workflows/self_host_build.yml index 8bb57a4b4..b728e33c3 100644 --- a/.github/workflows/self_host_build.yml +++ b/.github/workflows/self_host_build.yml @@ -23,6 +23,7 @@ jobs: windows-build: name: Build Windows runs-on: [self-hosted, windows, x64] + timeout-minutes: 10 strategy: matrix: config: @@ -71,6 +72,7 @@ jobs: linux-build: name: Build Linux runs-on: [self-hosted, linux, x64] + timeout-minutes: 10 strategy: matrix: config: -- cgit v1.2.3 From 3598016c6cf785d777c97055c14d186cd0b65ba2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 23 May 2022 23:42:05 +0200 Subject: create_release.yml --- .github/workflows/create_release.yml | 112 +++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 .github/workflows/create_release.yml diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml new file mode 100644 index 000000000..9951164c2 --- /dev/null +++ b/.github/workflows/create_release.yml @@ -0,0 +1,112 @@ +name: Create Release + +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + +jobs: + windows-bundle: + + runs-on: [self-hosted, windows, x64] + + strategy: + matrix: + config: + - 'release' + arch: + - 'x64' + + env: + VCPKG_VERSION: 2022.03.10 + + steps: + - uses: actions/checkout@v2 + + - name: Setup xmake + uses: xmake-io/github-action-setup-xmake@v1 + with: + xmake-version: 2.6.4 + + - name: Installing vcpkg + run: | + git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg + cd .vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg.exe integrate install + cd .. + + - name: Cache vcpkg + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}\.vcpkg\installed + key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + + - name: Bundle + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - uses: actions/upload-artifact@v3 + name: upload ${{matrix.target}} + with: + name: linux_x64-${{matrix.target}}-${{matrix.config}} + path: | + build/zenserver-win64.zip + + linux-bundle: + runs-on: [self-hosted, linux, x64] + + strategy: + matrix: + config: + - 'release' + arch: + - 'x64' + + env: + VCPKG_VERSION: 2022.03.10 + + steps: + - uses: actions/checkout@v2 + + - name: Set up GCC 11 + uses: egor-tensin/setup-gcc@v1 + with: + version: 11 + platform: x64 + + - name: Setup xmake + uses: xmake-io/github-action-setup-xmake@v1 + with: + xmake-version: 2.6.4 + + - name: Installing vcpkg + run: | + git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg + cd .vcpkg + ./bootstrap-vcpkg.sh + cd .. + + - name: Cache vcpkg + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/.vcpkg/installed + key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + + - name: Bundle + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - uses: actions/upload-artifact@v3 + name: upload ${{matrix.target}} + with: + name: linux_x64-${{matrix.target}}-${{matrix.config}} + path: | + build/zenserver-linux.zip -- cgit v1.2.3 From fec1425199731ede0d66ecc05616c5fd7c4385a9 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 23 May 2022 23:55:23 +0200 Subject: fix 7zip and create release reading from CHANGELOG.md --- .github/workflows/create_release.yml | 77 ++++++++++++++++++++++++------------ CHANGELOG.md | 0 2 files changed, 51 insertions(+), 26 deletions(-) create mode 100644 CHANGELOG.md diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 9951164c2..eca84289f 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -7,17 +7,10 @@ on: - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 jobs: - windows-bundle: + bundle-windows: runs-on: [self-hosted, windows, x64] - strategy: - matrix: - config: - - 'release' - arch: - - 'x64' - env: VCPKG_VERSION: 2022.03.10 @@ -42,7 +35,11 @@ jobs: with: path: | ${{ github.workspace }}\.vcpkg\installed - key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 + + - name: Update path for 7zip + run: | + echo "C:\'Program Files'\7-Zip\7z.exe" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - name: Bundle run: | @@ -51,22 +48,14 @@ jobs: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - uses: actions/upload-artifact@v3 - name: upload ${{matrix.target}} + name: upload artifacts with: - name: linux_x64-${{matrix.target}}-${{matrix.config}} - path: | - build/zenserver-win64.zip + name: zenserver-win64 + path: build/zenserver-win64.zip - linux-bundle: + bundle-linux: runs-on: [self-hosted, linux, x64] - strategy: - matrix: - config: - - 'release' - arch: - - 'x64' - env: VCPKG_VERSION: 2022.03.10 @@ -96,7 +85,7 @@ jobs: with: path: | ${{ github.workspace }}/.vcpkg/installed - key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - name: Bundle run: | @@ -105,8 +94,44 @@ jobs: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - uses: actions/upload-artifact@v3 - name: upload ${{matrix.target}} + name: upload zenserver-linux with: - name: linux_x64-${{matrix.target}}-${{matrix.config}} - path: | - build/zenserver-linux.zip + name: zenserver-linux + path: build/zenserver-linux.zip + + create-release: + runs-on: ubuntu-latest + needs: [bundle-linux, bundle-win32] + steps: + - name: Download Linux artifacts + uses: actions/download-artifact@v1 + with: + name: zenserver-linux + path: zenserver-linux.zip + + - name: Download Windows artifacts + uses: actions/download-artifact@v1 + with: + name: zenserver-win64 + path: zenserver-win64.zip + + - name: Check prerelease + id: get-prerelease + uses: haya14busa/action-cond@v1 + with: + cond: ${{contains(github.ref, '-pre')}} + if_true: "true" + if_false: "false" + + - name: Create Release + id: create_release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{github.ref.name}} + body: | + ${{steps.read_changelog.outputs.contents}} + draft: false + prerelease: ${{steps.get-prerelease.outputs.value}} + files: "*.zip" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..e69de29bb -- cgit v1.2.3 From 02ec64f356b230f528d545150e64e6d1a20c95d4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 23 May 2022 23:56:22 +0200 Subject: fix create-release dependency --- .github/workflows/create_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index eca84289f..c0ff18749 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -101,7 +101,7 @@ jobs: create-release: runs-on: ubuntu-latest - needs: [bundle-linux, bundle-win32] + needs: [bundle-linux, bundle-windows] steps: - name: Download Linux artifacts uses: actions/download-artifact@v1 -- cgit v1.2.3 From 2e9a32cf80cdea53689d376f57309780cb9a83aa Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 23 May 2022 23:59:57 +0200 Subject: bad 7zip path --- .github/workflows/create_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index c0ff18749..ec0261fdf 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -39,7 +39,7 @@ jobs: - name: Update path for 7zip run: | - echo "C:\'Program Files'\7-Zip\7z.exe" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "C:\'Program Files'\7-Zip" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - name: Bundle run: | -- cgit v1.2.3 From 6beda1db6987bbce95b87e44f589d66391513935 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 00:06:56 +0200 Subject: 7z path --- .github/workflows/create_release.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index ec0261fdf..91aced35f 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -15,6 +15,11 @@ jobs: VCPKG_VERSION: 2022.03.10 steps: + - name: Update path for 7zip + run: | + echo "C:\Program Files\7-Zip" >> $GITHUB_PATH + 7z + - uses: actions/checkout@v2 - name: Setup xmake @@ -37,10 +42,6 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - - name: Update path for 7zip - run: | - echo "C:\'Program Files'\7-Zip" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - name: Bundle run: | xmake bundle -v -y -- cgit v1.2.3 From 87c168c144feccdeaec0afc47403eaca19938599 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 00:13:33 +0200 Subject: skip windows until we can run 7z --- .github/workflows/create_release.yml | 100 +++++++++++++++++------------------ 1 file changed, 48 insertions(+), 52 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 91aced35f..97fd87d1e 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -7,52 +7,47 @@ on: - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 jobs: - bundle-windows: - - runs-on: [self-hosted, windows, x64] - - env: - VCPKG_VERSION: 2022.03.10 - - steps: - - name: Update path for 7zip - run: | - echo "C:\Program Files\7-Zip" >> $GITHUB_PATH - 7z - - - uses: actions/checkout@v2 - - - name: Setup xmake - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: 2.6.4 - - - name: Installing vcpkg - run: | - git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg - cd .vcpkg - .\bootstrap-vcpkg.bat - .\vcpkg.exe integrate install - cd .. - - - name: Cache vcpkg - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}\.vcpkg\installed - key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - - - name: Bundle - run: | - xmake bundle -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - uses: actions/upload-artifact@v3 - name: upload artifacts - with: - name: zenserver-win64 - path: build/zenserver-win64.zip +# bundle-windows: +# +# runs-on: [self-hosted, windows, x64] +# +# env: +# VCPKG_VERSION: 2022.03.10 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Setup xmake +# uses: xmake-io/github-action-setup-xmake@v1 +# with: +# xmake-version: 2.6.4 +# +# - name: Installing vcpkg +# run: | +# git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg +# cd .vcpkg +# .\bootstrap-vcpkg.bat +# .\vcpkg.exe integrate install +# cd .. +# +# - name: Cache vcpkg +# uses: actions/cache@v2 +# with: +# path: | +# ${{ github.workspace }}\.vcpkg\installed +# key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 +# +# - name: Bundle +# run: | +# xmake bundle -v -y +# env: +# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg +# +# - uses: actions/upload-artifact@v3 +# name: upload artifacts +# with: +# name: zenserver-win64 +# path: build/zenserver-win64.zip bundle-linux: runs-on: [self-hosted, linux, x64] @@ -102,7 +97,8 @@ jobs: create-release: runs-on: ubuntu-latest - needs: [bundle-linux, bundle-windows] +# needs: [bundle-linux, bundle-windows] + needs: [bundle-linux] steps: - name: Download Linux artifacts uses: actions/download-artifact@v1 @@ -110,11 +106,11 @@ jobs: name: zenserver-linux path: zenserver-linux.zip - - name: Download Windows artifacts - uses: actions/download-artifact@v1 - with: - name: zenserver-win64 - path: zenserver-win64.zip +# - name: Download Windows artifacts +# uses: actions/download-artifact@v1 +# with: +# name: zenserver-win64 +# path: zenserver-win64.zip - name: Check prerelease id: get-prerelease -- cgit v1.2.3 From 954a1be4fe2e0ab872f04954bd747ae043e4363a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 00:19:00 +0200 Subject: fix path to release artifacts upload --- .github/workflows/create_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 97fd87d1e..8baf3ff3a 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -131,4 +131,4 @@ jobs: ${{steps.read_changelog.outputs.contents}} draft: false prerelease: ${{steps.get-prerelease.outputs.value}} - files: "*.zip" + files: "zen/*.zip" -- cgit v1.2.3 From f51ee0cd76381f97307f8de10a17cfe99468b638 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 00:40:11 +0200 Subject: find downloaded artifacts --- .github/workflows/create_release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 8baf3ff3a..7a4999af4 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -105,6 +105,9 @@ jobs: with: name: zenserver-linux path: zenserver-linux.zip + - run: | + pwd + ls -la # - name: Download Windows artifacts # uses: actions/download-artifact@v1 -- cgit v1.2.3 From ca25f8759b83d73149d8d459d75df4db50ef1135 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 00:48:04 +0200 Subject: artifact path --- .github/workflows/create_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 7a4999af4..964d76cd2 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -134,4 +134,4 @@ jobs: ${{steps.read_changelog.outputs.contents}} draft: false prerelease: ${{steps.get-prerelease.outputs.value}} - files: "zen/*.zip" + files: "*.zip" -- cgit v1.2.3 From 71fe5835e05dc1d9cc73e52eb1ae48a5900632f0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 01:00:48 +0200 Subject: no wildcards --- .github/workflows/create_release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 964d76cd2..2858b0138 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -134,4 +134,6 @@ jobs: ${{steps.read_changelog.outputs.contents}} draft: false prerelease: ${{steps.get-prerelease.outputs.value}} - files: "*.zip" + files: | + "zenserver-linux.zip" +# "zenserver-win64.zip" -- cgit v1.2.3 From 0d408e86a4b34e5ee9139d687f30026af5fbe3ee Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 08:53:25 +0200 Subject: absolute paths --- .github/workflows/create_release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 2858b0138..917687c5a 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -104,7 +104,7 @@ jobs: uses: actions/download-artifact@v1 with: name: zenserver-linux - path: zenserver-linux.zip + path: ${{ github.workspace }}/zenserver-linux.zip - run: | pwd ls -la @@ -113,7 +113,7 @@ jobs: # uses: actions/download-artifact@v1 # with: # name: zenserver-win64 -# path: zenserver-win64.zip +# path: ${{ github.workspace }}/zenserver-win64.zip - name: Check prerelease id: get-prerelease @@ -135,5 +135,5 @@ jobs: draft: false prerelease: ${{steps.get-prerelease.outputs.value}} files: | - "zenserver-linux.zip" -# "zenserver-win64.zip" + "${{ github.workspace }}/zenserver-linux.zip" +# "${{ github.workspace }}/zenserver-win64.zip" -- cgit v1.2.3 From 3920e5742af0b47d7b290186e4abc3ef2b8c11c4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 08:58:08 +0200 Subject: remove quotes --- .github/workflows/create_release.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 917687c5a..66a6865f5 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -104,7 +104,7 @@ jobs: uses: actions/download-artifact@v1 with: name: zenserver-linux - path: ${{ github.workspace }}/zenserver-linux.zip + path: zenserver-linux.zip - run: | pwd ls -la @@ -113,7 +113,7 @@ jobs: # uses: actions/download-artifact@v1 # with: # name: zenserver-win64 -# path: ${{ github.workspace }}/zenserver-win64.zip +# path: zenserver-win64.zip - name: Check prerelease id: get-prerelease @@ -135,5 +135,6 @@ jobs: draft: false prerelease: ${{steps.get-prerelease.outputs.value}} files: | - "${{ github.workspace }}/zenserver-linux.zip" -# "${{ github.workspace }}/zenserver-win64.zip" + zenserver-linux.zip + zenserver-win64.zip + -- cgit v1.2.3 From 1bc36a8525010926ea4d4e93fccba8e67c465e16 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 09:04:54 +0200 Subject: get artifact target name is a directory --- .github/workflows/create_release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 66a6865f5..d1e2598c5 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -104,7 +104,7 @@ jobs: uses: actions/download-artifact@v1 with: name: zenserver-linux - path: zenserver-linux.zip + path: artifacts - run: | pwd ls -la @@ -113,7 +113,7 @@ jobs: # uses: actions/download-artifact@v1 # with: # name: zenserver-win64 -# path: zenserver-win64.zip +# path: artifacts - name: Check prerelease id: get-prerelease @@ -135,6 +135,6 @@ jobs: draft: false prerelease: ${{steps.get-prerelease.outputs.value}} files: | - zenserver-linux.zip - zenserver-win64.zip + artifacts/zenserver-linux.zip +# artifacts/zenserver-win64.zip -- cgit v1.2.3 From 3036546fa568b63b94f50e44d69726e244e5e890 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 09:11:14 +0200 Subject: read release changelog from CHANGELOG.md --- .github/workflows/create_release.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index d1e2598c5..787892587 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -104,16 +104,13 @@ jobs: uses: actions/download-artifact@v1 with: name: zenserver-linux - path: artifacts - - run: | - pwd - ls -la + path: linux # - name: Download Windows artifacts # uses: actions/download-artifact@v1 # with: # name: zenserver-win64 -# path: artifacts +# path: win64 - name: Check prerelease id: get-prerelease @@ -123,6 +120,12 @@ jobs: if_true: "true" if_false: "false" + - name: Read CHANGELOG + id: read_changelog + uses: andstor/file-reader-action@v1 + with: + path: "CHANGELOG.md" + - name: Create Release id: create_release uses: softprops/action-gh-release@v1 @@ -135,6 +138,6 @@ jobs: draft: false prerelease: ${{steps.get-prerelease.outputs.value}} files: | - artifacts/zenserver-linux.zip -# artifacts/zenserver-win64.zip + linux/zenserver-linux.zip +# win64/zenserver-win64.zip -- cgit v1.2.3 From 77ed706085e832a8d27397a7285501c35e2fbf63 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 09:15:20 +0200 Subject: Update CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29bb..9c57ca701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +- Add release flow in GitHub actions on pushed tag `v0.1.2` gives full release, `v0.1.2-pre0` gives pre-release +- Namespaces: This introduces namespaces to the zenserver but only the default ue4.ddc is supported. Clients that don't send a namespace in the request will keep old behviour, new clients that sends namespace is required to use ue4.ddc (which they currently do) +- Aligned bucket naming rules with UE code base +- Fix retry counter and add an extra iteration to give more time for success during contention for standalone files in cache +- Make sure CacheBucket::PutStandaloneCacheValue cleans up the temp file +- Restore logic where we accept failed overwrite if resulting size is the same for standlone file in cache +- Correctly calculate the m_TotalSize difference when overwriting file for standalone files in cache +- Fix namespace folder scanning -- cgit v1.2.3 From f30857a0c38d265a9d677cc6f0956dd65f89a4f6 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 May 2022 09:20:31 +0200 Subject: check out repo to get to CHANGELOG.md --- .github/workflows/create_release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 787892587..bfc5d612b 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -100,6 +100,8 @@ jobs: # needs: [bundle-linux, bundle-windows] needs: [bundle-linux] steps: + - uses: actions/checkout@v2 + - name: Download Linux artifacts uses: actions/download-artifact@v1 with: -- cgit v1.2.3 From f45bd3e1f975fb3ce64eccea62563967b0ca94de Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 20:06:16 +0200 Subject: cleanup --- .github/workflows/create_release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index bfc5d612b..9ba83aeda 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -43,8 +43,8 @@ jobs: # env: # VCPKG_ROOT: ${{ github.workspace }}/.vcpkg # -# - uses: actions/upload-artifact@v3 -# name: upload artifacts +# . name: zenserver-win64 +# uses: actions/upload-artifact@v3 # with: # name: zenserver-win64 # path: build/zenserver-win64.zip @@ -89,8 +89,8 @@ jobs: env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - uses: actions/upload-artifact@v3 - name: upload zenserver-linux + - name: upload zenserver-linux + uses: actions/upload-artifact@v3 with: name: zenserver-linux path: build/zenserver-linux.zip -- cgit v1.2.3 From 67954fa86e56ebd01edefb8b0835b418c85f04ce Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 20:06:35 +0200 Subject: bundle if we are on main branch --- .github/workflows/self_host_build.yml | 36 +++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/self_host_build.yml b/.github/workflows/self_host_build.yml index b728e33c3..933fcd321 100644 --- a/.github/workflows/self_host_build.yml +++ b/.github/workflows/self_host_build.yml @@ -1,8 +1,8 @@ -name: Validate Build +name: Validate on: pull_request: - types: [opened, reopened, synchronize, reopened] + types: [opened, reopened, synchronize] branches: [ main ] jobs: @@ -21,7 +21,7 @@ jobs: exclude-regex: (.*thirdparty.*) windows-build: - name: Build Windows + name: Build & Test Windows runs-on: [self-hosted, windows, x64] timeout-minutes: 10 strategy: @@ -57,6 +57,20 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + - name: Bundle + if: ${{ github.ref_name == 'main') + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: upload zenserver-linux + if: ${{ github.ref_name == 'main') + uses: actions/upload-artifact@v3 + with: + name: zenserver-linux + path: build/zenserver-linux.zip + - name: Config run: | xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} @@ -70,7 +84,7 @@ jobs: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg linux-build: - name: Build Linux + name: Build & Test Linux runs-on: [self-hosted, linux, x64] timeout-minutes: 10 strategy: @@ -111,6 +125,20 @@ jobs: ${{ github.workspace }}/.vcpkg/installed key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + - name: Bundle + if: ${{ github.ref_name == 'main') + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: upload zenserver-win64 + if: ${{ github.ref_name == 'main') + uses: actions/upload-artifact@v3 + with: + name: zenserver-win64 + path: build/zenserver-win64.zip + - name: Config run: | xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} -- cgit v1.2.3 From f3ff618c4dc8041f952816264cb785ad39cb3910 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 20:07:57 +0200 Subject: use tagging as trigger for creating release --- .github/workflows/self_host_build.yml | 152 ---------------------------------- .github/workflows/update_release.yml | 134 ------------------------------ .github/workflows/validate.yml | 152 ++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 286 deletions(-) delete mode 100644 .github/workflows/self_host_build.yml delete mode 100644 .github/workflows/update_release.yml create mode 100644 .github/workflows/validate.yml diff --git a/.github/workflows/self_host_build.yml b/.github/workflows/self_host_build.yml deleted file mode 100644 index 933fcd321..000000000 --- a/.github/workflows/self_host_build.yml +++ /dev/null @@ -1,152 +0,0 @@ -name: Validate - -on: - pull_request: - types: [opened, reopened, synchronize] - branches: [ main ] - -jobs: - clang-format: - name: Check clang-format - runs-on: [self-hosted, linux, x64] - - steps: - - uses: actions/checkout@v2 - - - name: clang-format - uses: jidicula/clang-format-action@v4.6.2 - with: - clang-format-version: '13' - check-path: '.' - exclude-regex: (.*thirdparty.*) - - windows-build: - name: Build & Test Windows - runs-on: [self-hosted, windows, x64] - timeout-minutes: 10 - strategy: - matrix: - config: - - 'debug' - - 'release' - arch: - - 'x64' - env: - VCPKG_VERSION: 2022.03.10 - - steps: - - uses: actions/checkout@v2 - - - name: Setup xmake - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: 2.6.4 - - - name: Installing vcpkg - run: | - git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg - cd .vcpkg - .\bootstrap-vcpkg.bat - .\vcpkg.exe integrate install - cd .. - - - name: Cache vcpkg - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}\.vcpkg\installed - key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - - - name: Bundle - if: ${{ github.ref_name == 'main') - run: | - xmake bundle -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: upload zenserver-linux - if: ${{ github.ref_name == 'main') - uses: actions/upload-artifact@v3 - with: - name: zenserver-linux - path: build/zenserver-linux.zip - - - name: Config - run: | - xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: Build & Test - run: | - xmake test -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - linux-build: - name: Build & Test Linux - runs-on: [self-hosted, linux, x64] - timeout-minutes: 10 - strategy: - matrix: - config: - - 'debug' - - 'release' - arch: - - 'x86_64' - env: - VCPKG_VERSION: 2022.03.10 - - steps: - - uses: actions/checkout@v2 - - - name: Set up GCC 11 - uses: egor-tensin/setup-gcc@v1 - with: - version: 11 - platform: x64 - - - name: Setup xmake - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: 2.6.4 - - - name: Installing vcpkg - run: | - git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg - cd .vcpkg - ./bootstrap-vcpkg.sh - cd .. - - - name: Cache vcpkg - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/.vcpkg/installed - key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - - - name: Bundle - if: ${{ github.ref_name == 'main') - run: | - xmake bundle -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: upload zenserver-win64 - if: ${{ github.ref_name == 'main') - uses: actions/upload-artifact@v3 - with: - name: zenserver-win64 - path: build/zenserver-win64.zip - - - name: Config - run: | - xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: Build & Test - run: | - xmake test -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg diff --git a/.github/workflows/update_release.yml b/.github/workflows/update_release.yml deleted file mode 100644 index 27d5e2783..000000000 --- a/.github/workflows/update_release.yml +++ /dev/null @@ -1,134 +0,0 @@ -name: Build release - -on: - # push - pull_request: - types: [closed] - branches: [ main ] - -jobs: - windows-build: - if: >- - github.event.pull_request.merged == true && - contains( github.event.pull_request.labels.*.name, 'release') - name: Build Windows - runs-on: [self-hosted, windows, x64] - strategy: - matrix: - config: - - 'release' - arch: - - 'x64' - env: - VCPKG_VERSION: 2022.03.10 - - steps: - - uses: actions/checkout@v2 - - - name: Setup xmake - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: 2.6.4 - - - name: Installing vcpkg - run: | - git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg - cd .vcpkg - .\bootstrap-vcpkg.bat - .\vcpkg.exe integrate install - cd .. - - - name: Cache vcpkg - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}\.vcpkg\installed - key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - - - name: Config - run: | - xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: Build - run: | - xmake build -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - # - name: Create Archive - # run: | - # cd .\build\windows\${{ matrix.arch }}\${{ matrix.config }} - # C:\'Program Files'\7-Zip\7z.exe a -r ..\..\..\..\windows-${{ matrix.arch }}-${{ matrix.config }}.zip * - # cd ..\..\..\.. - - - name: Create Archive - run: | - cd .\build\windows\${{ matrix.arch }}\${{ matrix.config }} - C:\'Program Files'\7-Zip\7z.exe a -r ..\..\..\..\zenserver-win64.zip zenserver.exe - cd ..\..\..\.. - - - name: Get current release version info - run: | - $repo = "EpicGames/zen" - $releases = "https://api.github.com/repos/$repo/releases/latest" - Write-Host Determining latest release - $latest = (Invoke-WebRequest -Headers @{"Accept"="application/vnd.github.v3+json";"Authorization"="token ${{ secrets.GITHUB_TOKEN }}"} $releases | ConvertFrom-Json)[0] - $current_version_tag = [version]$latest.tag_name.replace('v','') - echo "Current version" $current_version_tag - if ($current_version_tag.Revision.Equals(9)) { - if ($current_version_tag.Build.Equals(9)) { - $new_version_tag = [version]::New($current_version_tag.Major,$current_version_tag.Minor+1,0,0).toString() - }else { - $new_version_tag = [version]::New($current_version_tag.Major,$current_version_tag.Minor,$current_version_tag.Build+1,0).toString() - } - }else { - $new_version_tag = [version]::New($current_version_tag.Major,$current_version_tag.Minor,$current_version_tag.Build,$current_version_tag.Revision+1).toString() - } - echo $new_version_tag - echo "new_version_tag=$new_version_tag" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: v${{ env.new_version_tag }} - release_name: Release - draft: false - prerelease: false - - # - name: Create Release - # id: create_release - # uses: actions/create-release@v1 - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # with: - # tag_name: ${{ github.ref_name }} - # release_name: Release ${{ github.head_ref }} - # draft: false - # prerelease: false - - # - name: Upload Release Asset - # id: upload-release-asset - # uses: actions/upload-release-asset@v1 - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # with: - # upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - # asset_path: .\windows-${{ matrix.arch }}-${{ matrix.config }}.zip - # asset_name: windows-${{ matrix.arch }}-${{ matrix.config }} - # asset_content_type: application/zip - - name: Upload Release Asset - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: .\zenserver-win64.zip - asset_name: zenserver-win64.zip - asset_content_type: application/zip - diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 000000000..933fcd321 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,152 @@ +name: Validate + +on: + pull_request: + types: [opened, reopened, synchronize] + branches: [ main ] + +jobs: + clang-format: + name: Check clang-format + runs-on: [self-hosted, linux, x64] + + steps: + - uses: actions/checkout@v2 + + - name: clang-format + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: '.' + exclude-regex: (.*thirdparty.*) + + windows-build: + name: Build & Test Windows + runs-on: [self-hosted, windows, x64] + timeout-minutes: 10 + strategy: + matrix: + config: + - 'debug' + - 'release' + arch: + - 'x64' + env: + VCPKG_VERSION: 2022.03.10 + + steps: + - uses: actions/checkout@v2 + + - name: Setup xmake + uses: xmake-io/github-action-setup-xmake@v1 + with: + xmake-version: 2.6.4 + + - name: Installing vcpkg + run: | + git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg + cd .vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg.exe integrate install + cd .. + + - name: Cache vcpkg + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}\.vcpkg\installed + key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + + - name: Bundle + if: ${{ github.ref_name == 'main') + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: upload zenserver-linux + if: ${{ github.ref_name == 'main') + uses: actions/upload-artifact@v3 + with: + name: zenserver-linux + path: build/zenserver-linux.zip + + - name: Config + run: | + xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: Build & Test + run: | + xmake test -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + linux-build: + name: Build & Test Linux + runs-on: [self-hosted, linux, x64] + timeout-minutes: 10 + strategy: + matrix: + config: + - 'debug' + - 'release' + arch: + - 'x86_64' + env: + VCPKG_VERSION: 2022.03.10 + + steps: + - uses: actions/checkout@v2 + + - name: Set up GCC 11 + uses: egor-tensin/setup-gcc@v1 + with: + version: 11 + platform: x64 + + - name: Setup xmake + uses: xmake-io/github-action-setup-xmake@v1 + with: + xmake-version: 2.6.4 + + - name: Installing vcpkg + run: | + git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg + cd .vcpkg + ./bootstrap-vcpkg.sh + cd .. + + - name: Cache vcpkg + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/.vcpkg/installed + key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 + + - name: Bundle + if: ${{ github.ref_name == 'main') + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: upload zenserver-win64 + if: ${{ github.ref_name == 'main') + uses: actions/upload-artifact@v3 + with: + name: zenserver-win64 + path: build/zenserver-win64.zip + + - name: Config + run: | + xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: Build & Test + run: | + xmake test -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -- cgit v1.2.3 From 43629e7c0fcba047c03bf10710488dd1453c9d64 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 25 May 2022 20:10:31 +0200 Subject: fix brackets --- .github/workflows/validate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 933fcd321..e2f5d0ef2 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -58,14 +58,14 @@ jobs: key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - name: Bundle - if: ${{ github.ref_name == 'main') + if: ${{ github.ref_name == 'main'}} run: | xmake bundle -v -y env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - name: upload zenserver-linux - if: ${{ github.ref_name == 'main') + if: ${{ github.ref_name == 'main'}} uses: actions/upload-artifact@v3 with: name: zenserver-linux @@ -126,14 +126,14 @@ jobs: key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - name: Bundle - if: ${{ github.ref_name == 'main') + if: ${{ github.ref_name == 'main'}} run: | xmake bundle -v -y env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - name: upload zenserver-win64 - if: ${{ github.ref_name == 'main') + if: ${{ github.ref_name == 'main'}} uses: actions/upload-artifact@v3 with: name: zenserver-win64 -- cgit v1.2.3 From 681336cd088320b374d20b47370179073179ad0c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:07:46 +0200 Subject: re-enable windows bundle on create release --- .github/workflows/create_release.yml | 100 +++++++++++++++++------------------ 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 9ba83aeda..317c8e322 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -7,47 +7,47 @@ on: - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 jobs: -# bundle-windows: -# -# runs-on: [self-hosted, windows, x64] -# -# env: -# VCPKG_VERSION: 2022.03.10 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Setup xmake -# uses: xmake-io/github-action-setup-xmake@v1 -# with: -# xmake-version: 2.6.4 -# -# - name: Installing vcpkg -# run: | -# git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg -# cd .vcpkg -# .\bootstrap-vcpkg.bat -# .\vcpkg.exe integrate install -# cd .. -# -# - name: Cache vcpkg -# uses: actions/cache@v2 -# with: -# path: | -# ${{ github.workspace }}\.vcpkg\installed -# key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 -# -# - name: Bundle -# run: | -# xmake bundle -v -y -# env: -# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -# -# . name: zenserver-win64 -# uses: actions/upload-artifact@v3 -# with: -# name: zenserver-win64 -# path: build/zenserver-win64.zip + bundle-windows: + + runs-on: [self-hosted, windows, x64] + + env: + VCPKG_VERSION: 2022.03.10 + + steps: + - uses: actions/checkout@v2 + + - name: Setup xmake + uses: xmake-io/github-action-setup-xmake@v1 + with: + xmake-version: 2.6.4 + + - name: Installing vcpkg + run: | + git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg + cd .vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg.exe integrate install + cd .. + + - name: Cache vcpkg + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}\.vcpkg\installed + key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 + + - name: Bundle + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: Upload zenserver-win64 + uses: actions/upload-artifact@v3 + with: + name: zenserver-win64 + path: build/zenserver-win64.zip bundle-linux: runs-on: [self-hosted, linux, x64] @@ -89,7 +89,7 @@ jobs: env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - name: upload zenserver-linux + - name: Upload zenserver-linux uses: actions/upload-artifact@v3 with: name: zenserver-linux @@ -97,8 +97,7 @@ jobs: create-release: runs-on: ubuntu-latest -# needs: [bundle-linux, bundle-windows] - needs: [bundle-linux] + needs: [bundle-linux, bundle-windows] steps: - uses: actions/checkout@v2 @@ -108,11 +107,11 @@ jobs: name: zenserver-linux path: linux -# - name: Download Windows artifacts -# uses: actions/download-artifact@v1 -# with: -# name: zenserver-win64 -# path: win64 + - name: Download Windows artifacts + uses: actions/download-artifact@v1 + with: + name: zenserver-win64 + path: win64 - name: Check prerelease id: get-prerelease @@ -141,5 +140,4 @@ jobs: prerelease: ${{steps.get-prerelease.outputs.value}} files: | linux/zenserver-linux.zip -# win64/zenserver-win64.zip - + win64/zenserver-win64.zip -- cgit v1.2.3 From dfd6e4c5461866a6e8edd5ebdc2a9a1ab21cf4de Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:20:16 +0200 Subject: clear xmake cache --- .github/workflows/create_release.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 317c8e322..dda893025 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -22,6 +22,9 @@ jobs: with: xmake-version: 2.6.4 + - name: Clear xmake Cache + run: xmake f -c + - name: Installing vcpkg run: | git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg @@ -69,6 +72,9 @@ jobs: with: xmake-version: 2.6.4 + - name: Clear xmake Cache + run: xmake f -c + - name: Installing vcpkg run: | git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg -- cgit v1.2.3 From fabd658afde391f2cac5795257635cb69807aae5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:22:34 +0200 Subject: clear xmake cache take 2 --- .github/workflows/create_release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index dda893025..9ce08da26 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -22,9 +22,6 @@ jobs: with: xmake-version: 2.6.4 - - name: Clear xmake Cache - run: xmake f -c - - name: Installing vcpkg run: | git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg @@ -40,6 +37,9 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 + - name: Clear xmake Cache + run: xmake f -c -y + - name: Bundle run: | xmake bundle -v -y @@ -72,9 +72,6 @@ jobs: with: xmake-version: 2.6.4 - - name: Clear xmake Cache - run: xmake f -c - - name: Installing vcpkg run: | git clone -b ${{env.VCPKG_VERSION}} --single-branch https://github.com/Microsoft/vcpkg.git .vcpkg @@ -89,6 +86,9 @@ jobs: ${{ github.workspace }}/.vcpkg/installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 + - name: Clear xmake Cache + run: xmake f -c -y + - name: Bundle run: | xmake bundle -v -y -- cgit v1.2.3 From 47825e8ebbeeeb03d500059a5ef9740cda7d80a8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:24:25 +0200 Subject: clear build folder --- .github/workflows/create_release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 9ce08da26..334832098 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -37,8 +37,8 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - - name: Clear xmake Cache - run: xmake f -c -y + - name: Clear build folder + run: rm -rf build - name: Bundle run: | @@ -86,8 +86,8 @@ jobs: ${{ github.workspace }}/.vcpkg/installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - - name: Clear xmake Cache - run: xmake f -c -y + - name: Clear build folder + run: rm -rf build - name: Bundle run: | -- cgit v1.2.3 From 909f72badd96779d2e73c4f48af83f9fe413704f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:27:01 +0200 Subject: clear build folder with Powershell - temp disable validate --- .github/workflows/create_release.yml | 2 +- .github/workflows/validate.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 334832098..aa269f5dd 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -38,7 +38,7 @@ jobs: key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - name: Clear build folder - run: rm -rf build + run: Remove-Item -Recurse build - name: Bundle run: | diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e2f5d0ef2..9c163a268 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -2,7 +2,7 @@ name: Validate on: pull_request: - types: [opened, reopened, synchronize] + types: [opened, reopened] #, synchronize] branches: [ main ] jobs: -- cgit v1.2.3 From 841f2be52cdde46f4614dc8d7225f54a68a641e0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:29:47 +0200 Subject: remove clear build folder --- .github/workflows/create_release.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index aa269f5dd..317c8e322 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -37,9 +37,6 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - - name: Clear build folder - run: Remove-Item -Recurse build - - name: Bundle run: | xmake bundle -v -y @@ -86,9 +83,6 @@ jobs: ${{ github.workspace }}/.vcpkg/installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - - name: Clear build folder - run: rm -rf build - - name: Bundle run: | xmake bundle -v -y -- cgit v1.2.3 From fe8426b6c705d7e4a2f72edc8b7a5cb39285bd14 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:34:57 +0200 Subject: show path --- .github/workflows/create_release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 317c8e322..6827030c1 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -15,6 +15,8 @@ jobs: VCPKG_VERSION: 2022.03.10 steps: + - run: echo ${env:PATH} + - uses: actions/checkout@v2 - name: Setup xmake -- cgit v1.2.3 From 6b0c71806d9e0559880855591d338c70dca19fbf Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:39:11 +0200 Subject: go back to skipping windows bundle on release --- .github/workflows/create_release.yml | 34 +++++++++++++++++----------------- .github/workflows/validate.yml | 12 +++++++----- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 6827030c1..fb6d41ff9 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -39,17 +39,17 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 - - name: Bundle - run: | - xmake bundle -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: Upload zenserver-win64 - uses: actions/upload-artifact@v3 - with: - name: zenserver-win64 - path: build/zenserver-win64.zip +# - name: Bundle +# run: | +# xmake bundle -v -y +# env: +# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg +# +# - name: Upload zenserver-win64 +# uses: actions/upload-artifact@v3 +# with: +# name: zenserver-win64 +# path: build/zenserver-win64.zip bundle-linux: runs-on: [self-hosted, linux, x64] @@ -109,11 +109,11 @@ jobs: name: zenserver-linux path: linux - - name: Download Windows artifacts - uses: actions/download-artifact@v1 - with: - name: zenserver-win64 - path: win64 +# - name: Download Windows artifacts +# uses: actions/download-artifact@v1 +# with: +# name: zenserver-win64 +# path: win64 - name: Check prerelease id: get-prerelease @@ -142,4 +142,4 @@ jobs: prerelease: ${{steps.get-prerelease.outputs.value}} files: | linux/zenserver-linux.zip - win64/zenserver-win64.zip +# win64/zenserver-win64.zip diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 9c163a268..59354161f 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -2,7 +2,7 @@ name: Validate on: pull_request: - types: [opened, reopened] #, synchronize] + types: [opened, reopened, synchronize] branches: [ main ] jobs: @@ -35,7 +35,8 @@ jobs: VCPKG_VERSION: 2022.03.10 steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - name: Setup xmake uses: xmake-io/github-action-setup-xmake@v1 @@ -64,7 +65,7 @@ jobs: env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - name: upload zenserver-linux + - name: Upload zenserver-linux if: ${{ github.ref_name == 'main'}} uses: actions/upload-artifact@v3 with: @@ -98,7 +99,8 @@ jobs: VCPKG_VERSION: 2022.03.10 steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - name: Set up GCC 11 uses: egor-tensin/setup-gcc@v1 @@ -132,7 +134,7 @@ jobs: env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - name: upload zenserver-win64 + - name: Upload zenserver-win64 if: ${{ github.ref_name == 'main'}} uses: actions/upload-artifact@v3 with: -- cgit v1.2.3 From 8314e9721e37ffe72cbb8bae2bc4680c64399a0a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:42:33 +0200 Subject: remove windows bundle cancel previous validate for same branch (not main) --- .github/workflows/validate.yml | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 59354161f..b715a8c63 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -6,6 +6,17 @@ on: branches: [ main ] jobs: + cancel-old-build: + name: Cancel previous builds + runs-on: [self-hosted, linux, x64] + + steps: + - name: Cancel Previous Runs + if: ${{ github.ref_name != 'main'}} + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + clang-format: name: Check clang-format runs-on: [self-hosted, linux, x64] @@ -58,19 +69,19 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - - name: Bundle - if: ${{ github.ref_name == 'main'}} - run: | - xmake bundle -v -y - env: - VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - - name: Upload zenserver-linux - if: ${{ github.ref_name == 'main'}} - uses: actions/upload-artifact@v3 - with: - name: zenserver-linux - path: build/zenserver-linux.zip +# - name: Bundle +# if: ${{ github.ref_name == 'main'}} +# run: | +# xmake bundle -v -y +# env: +# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg +# +# - name: Upload zenserver-linux +# if: ${{ github.ref_name == 'main'}} +# uses: actions/upload-artifact@v3 +# with: +# name: zenserver-linux +# path: build/zenserver-linux.zip - name: Config run: | -- cgit v1.2.3 From 521aa9be0595706312760abcefb4380b085e58fb Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 26 May 2022 00:48:32 +0200 Subject: make sure cancel old builds runs first --- .github/workflows/validate.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index b715a8c63..827411a6b 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -18,6 +18,7 @@ jobs: access_token: ${{ github.token }} clang-format: + needs: cancel-old-build name: Check clang-format runs-on: [self-hosted, linux, x64] @@ -32,6 +33,7 @@ jobs: exclude-regex: (.*thirdparty.*) windows-build: + needs: cancel-old-build name: Build & Test Windows runs-on: [self-hosted, windows, x64] timeout-minutes: 10 @@ -96,6 +98,7 @@ jobs: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg linux-build: + needs: cancel-old-build name: Build & Test Linux runs-on: [self-hosted, linux, x64] timeout-minutes: 10 -- cgit v1.2.3 From 516ebc332eb47d2f2d1da302f844f419e054a3eb Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 28 May 2022 00:31:49 +0200 Subject: re-enable windows bundle --- .github/workflows/create_release.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index fb6d41ff9..6827030c1 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -39,17 +39,17 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-release-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-x64-v5 -# - name: Bundle -# run: | -# xmake bundle -v -y -# env: -# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -# -# - name: Upload zenserver-win64 -# uses: actions/upload-artifact@v3 -# with: -# name: zenserver-win64 -# path: build/zenserver-win64.zip + - name: Bundle + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: Upload zenserver-win64 + uses: actions/upload-artifact@v3 + with: + name: zenserver-win64 + path: build/zenserver-win64.zip bundle-linux: runs-on: [self-hosted, linux, x64] @@ -109,11 +109,11 @@ jobs: name: zenserver-linux path: linux -# - name: Download Windows artifacts -# uses: actions/download-artifact@v1 -# with: -# name: zenserver-win64 -# path: win64 + - name: Download Windows artifacts + uses: actions/download-artifact@v1 + with: + name: zenserver-win64 + path: win64 - name: Check prerelease id: get-prerelease @@ -142,4 +142,4 @@ jobs: prerelease: ${{steps.get-prerelease.outputs.value}} files: | linux/zenserver-linux.zip -# win64/zenserver-win64.zip + win64/zenserver-win64.zip -- cgit v1.2.3 From 7dcfd0db67bc06e3c2a6fe9f757a3c3954949d89 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 28 May 2022 02:11:03 +0200 Subject: enable windows bundle in validate --- .github/workflows/validate.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 827411a6b..7f022c76c 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -71,19 +71,19 @@ jobs: ${{ github.workspace }}\.vcpkg\installed key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 -# - name: Bundle -# if: ${{ github.ref_name == 'main'}} -# run: | -# xmake bundle -v -y -# env: -# VCPKG_ROOT: ${{ github.workspace }}/.vcpkg -# -# - name: Upload zenserver-linux -# if: ${{ github.ref_name == 'main'}} -# uses: actions/upload-artifact@v3 -# with: -# name: zenserver-linux -# path: build/zenserver-linux.zip + - name: Bundle + if: ${{ github.ref_name == 'main'}} + run: | + xmake bundle -v -y + env: + VCPKG_ROOT: ${{ github.workspace }}/.vcpkg + + - name: Upload zenserver-linux + if: ${{ github.ref_name == 'main'}} + uses: actions/upload-artifact@v3 + with: + name: zenserver-linux + path: build/zenserver-linux.zip - name: Config run: | -- cgit v1.2.3 From 2782656a83e403da24d162743e020e4f72497a4f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 31 May 2022 00:21:15 +0200 Subject: read part of changelog allowing us to keep history --- .github/workflows/create_release.yml | 8 ++++++-- CHANGELOG.md | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 6827030c1..7c7bb06cf 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -123,11 +123,15 @@ jobs: if_true: "true" if_false: "false" - - name: Read CHANGELOG + - name: Extract Version Changes + run: | + sed '1,/^##/!d;/##/d' CHANGELOG.md > CHANGELOG.tmp + + - name: Read CHANGELOG.tmp id: read_changelog uses: andstor/file-reader-action@v1 with: - path: "CHANGELOG.md" + path: "CHANGELOG.tmp" - name: Create Release id: create_release diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c57ca701..14358d4fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +## - Add release flow in GitHub actions on pushed tag `v0.1.2` gives full release, `v0.1.2-pre0` gives pre-release - Namespaces: This introduces namespaces to the zenserver but only the default ue4.ddc is supported. Clients that don't send a namespace in the request will keep old behviour, new clients that sends namespace is required to use ue4.ddc (which they currently do) - Aligned bucket naming rules with UE code base @@ -6,3 +7,5 @@ - Restore logic where we accept failed overwrite if resulting size is the same for standlone file in cache - Correctly calculate the m_TotalSize difference when overwriting file for standalone files in cache - Fix namespace folder scanning + +## older releases go here -- cgit v1.2.3 From d8c314f9ce297154fa6f0e5df72dcaab5944a107 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 2 Jun 2022 01:06:16 +0200 Subject: Bump version and update changelog --- CHANGELOG.md | 19 ++++++++++++++++--- xmake.lua | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14358d4fe..4d8fae2ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ ## -- Add release flow in GitHub actions on pushed tag `v0.1.2` gives full release, `v0.1.2-pre0` gives pre-release +- BlockStore (small object store) Always block GC of current write block +- Make it possible to configure GC monitoring interval using `--gc-monitor-interval-seconds` +- Keep "reason" from upstream response so we can present it even if the request fails without outright error +- New GitHub Actions release flow - Add release flow in GitHub actions on pushed tag `v0.1.2` gives full release, `v0.1.2-pre0` gives pre-release + +## 0d08450 +- Fixes issue with broken Zen instances for legacy requests + +## 63f50b5 +- Enable FILE_SHARE_DELETE on standalone files in disk buckets - fixes Jira UE-154234 +- Make sure we can properly create the block file before assigning it for use - fixes Jira UE-154438 +- Horde execute compressed input blobs +- Drop namespace support +- Safer delete of cache buckets + +## dba8b36 - Namespaces: This introduces namespaces to the zenserver but only the default ue4.ddc is supported. Clients that don't send a namespace in the request will keep old behviour, new clients that sends namespace is required to use ue4.ddc (which they currently do) - Aligned bucket naming rules with UE code base - Fix retry counter and add an extra iteration to give more time for success during contention for standalone files in cache @@ -7,5 +22,3 @@ - Restore logic where we accept failed overwrite if resulting size is the same for standlone file in cache - Correctly calculate the m_TotalSize difference when overwriting file for standalone files in cache - Fix namespace folder scanning - -## older releases go here diff --git a/xmake.lua b/xmake.lua index e4d795fcf..9ffc94c65 100644 --- a/xmake.lua +++ b/xmake.lua @@ -1,6 +1,6 @@ -- Copyright Epic Games, Inc. All Rights Reserved. -set_version("0.1.0", { build = "%Y%m%d%H%M" }) +set_version("0.1.1", { build = "%Y%m%d%H%M" }) set_configvar("ZEN_SCHEMA_VERSION", 3) -- changed cas oplog format (p3rl) add_requires( -- cgit v1.2.3 From 484423bf906149a823c97b0a7d201e1e76b30d06 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 2 Jun 2022 01:08:34 +0200 Subject: trigger validation build on push to main --- .github/workflows/validate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 7f022c76c..eed3036b2 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -4,6 +4,8 @@ on: pull_request: types: [opened, reopened, synchronize] branches: [ main ] + push: + branches: [ main ] jobs: cancel-old-build: -- cgit v1.2.3 From 04d2660701e349315328df108ab322387bf20507 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 2 Jun 2022 01:20:02 +0200 Subject: fix bundle upload path --- .github/workflows/validate.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index eed3036b2..0b1d3db27 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -80,12 +80,12 @@ jobs: env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - name: Upload zenserver-linux + - name: Upload zenserver-win64 if: ${{ github.ref_name == 'main'}} uses: actions/upload-artifact@v3 with: - name: zenserver-linux - path: build/zenserver-linux.zip + name: zenserver-win64 + path: build/zenserver-win64.zip - name: Config run: | @@ -150,12 +150,12 @@ jobs: env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - - name: Upload zenserver-win64 + - name: Upload zenserver-linux if: ${{ github.ref_name == 'main'}} uses: actions/upload-artifact@v3 with: - name: zenserver-win64 - path: build/zenserver-win64.zip + name: zenserver-linux + path: build/zenserver-linux.zip - name: Config run: | -- cgit v1.2.3 From cd3b6f6faec3a7f262faea34a9bd7c23a9cc6852 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 2 Jun 2022 01:30:48 +0200 Subject: only bundle on release config --- .github/workflows/validate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 0b1d3db27..e25891855 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -74,14 +74,14 @@ jobs: key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - name: Bundle - if: ${{ github.ref_name == 'main'}} + if: ${{ github.ref_name == 'main' && matrix.config == 'relase' }} run: | xmake bundle -v -y env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - name: Upload zenserver-win64 - if: ${{ github.ref_name == 'main'}} + if: ${{ github.ref_name == 'main' && matrix.config == 'relase' }} uses: actions/upload-artifact@v3 with: name: zenserver-win64 @@ -144,14 +144,14 @@ jobs: key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - name: Bundle - if: ${{ github.ref_name == 'main'}} + if: ${{ github.ref_name == 'main' && matrix.config == 'relase' }} run: | xmake bundle -v -y env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - name: Upload zenserver-linux - if: ${{ github.ref_name == 'main'}} + if: ${{ github.ref_name == 'main' && matrix.config == 'relase' }} uses: actions/upload-artifact@v3 with: name: zenserver-linux -- cgit v1.2.3 From 80cc1128e9401e8cce591be361a2652295e7a201 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 2 Jun 2022 01:35:32 +0200 Subject: release spelling --- .github/workflows/validate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e25891855..3988a9dfb 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -74,14 +74,14 @@ jobs: key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - name: Bundle - if: ${{ github.ref_name == 'main' && matrix.config == 'relase' }} + if: ${{ github.ref_name == 'main' && matrix.config == 'release' }} run: | xmake bundle -v -y env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - name: Upload zenserver-win64 - if: ${{ github.ref_name == 'main' && matrix.config == 'relase' }} + if: ${{ github.ref_name == 'main' && matrix.config == 'release' }} uses: actions/upload-artifact@v3 with: name: zenserver-win64 @@ -144,14 +144,14 @@ jobs: key: ${{ runner.os }}-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v5 - name: Bundle - if: ${{ github.ref_name == 'main' && matrix.config == 'relase' }} + if: ${{ github.ref_name == 'main' && matrix.config == 'release' }} run: | xmake bundle -v -y env: VCPKG_ROOT: ${{ github.workspace }}/.vcpkg - name: Upload zenserver-linux - if: ${{ github.ref_name == 'main' && matrix.config == 'relase' }} + if: ${{ github.ref_name == 'main' && matrix.config == 'release' }} uses: actions/upload-artifact@v3 with: name: zenserver-linux -- cgit v1.2.3 From c64d730ce45aa032d6d974d0d1288c29a8873fda Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 2 Jun 2022 10:00:47 +0200 Subject: move release job to in-house linux agent --- .github/workflows/create_release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 7c7bb06cf..fba2ec1b1 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -15,8 +15,6 @@ jobs: VCPKG_VERSION: 2022.03.10 steps: - - run: echo ${env:PATH} - - uses: actions/checkout@v2 - name: Setup xmake @@ -98,7 +96,7 @@ jobs: path: build/zenserver-linux.zip create-release: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] needs: [bundle-linux, bundle-windows] steps: - uses: actions/checkout@v2 -- cgit v1.2.3