diff options
| author | Stefan Boberg <[email protected]> | 2021-09-26 22:05:16 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2021-09-26 22:05:16 +0200 |
| commit | 71367d759aea47588225537ae4b8ab93c461bfe6 (patch) | |
| tree | 4a3fc6f9ccce0ac4212841e77563aaf92e76df60 | |
| parent | Eliminated use of ATL in StructuredCacheStore implementation (diff) | |
| download | zen-71367d759aea47588225537ae4b8ab93c461bfe6.tar.xz zen-71367d759aea47588225537ae4b8ab93c461bfe6.zip | |
Removed unused KV cache related code
| -rw-r--r-- | zenserver/cache/cachestore.cpp | 252 | ||||
| -rw-r--r-- | zenserver/cache/cachestore.h | 84 | ||||
| -rw-r--r-- | zenserver/zenserver.vcxproj | 2 | ||||
| -rw-r--r-- | zenserver/zenserver.vcxproj.filters | 6 |
4 files changed, 0 insertions, 344 deletions
diff --git a/zenserver/cache/cachestore.cpp b/zenserver/cache/cachestore.cpp deleted file mode 100644 index 2fc253a07..000000000 --- a/zenserver/cache/cachestore.cpp +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "cachestore.h" - -#include <zencore/crc32.h> -#include <zencore/except.h> -#include <zencore/logging.h> -#include <zencore/windows.h> - -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/iobuffer.h> -#include <zencore/string.h> -#include <zencore/thread.h> -#include <zenstore/basicfile.h> -#include <zenstore/cas.h> -#include <zenstore/caslog.h> - -#include <fmt/core.h> -#include <concepts> -#include <filesystem> -#include <gsl/gsl-lite.hpp> -#include <unordered_map> - -#include <atlfile.h> - -using namespace zen; -using namespace fmt::literals; - -namespace UE { - -struct CorruptionTrailer -{ - enum - { - /** Arbitrary number used to identify corruption **/ - MagicConstant = 0x1e873d89 - }; - - uint32_t Magic = MagicConstant; - uint32_t Version = 1; - uint32_t CRCofPayload = 0; - uint32_t SizeOfPayload = 0; - - void Initialize(const void* Data, size_t Size) - { - CRCofPayload = zen::MemCrc32_Deprecated(Data, Size); - SizeOfPayload = (uint32_t)Size; - } -}; - -std::filesystem::path -GenerateDdcPath(std::string_view Key, std::filesystem::path& rootDir) -{ - std::filesystem::path FilePath = rootDir; - - std::string k8{Key}; - for (auto& c : k8) - c = (char)toupper(c); - - const uint32_t Hash = zen::StrCrc_Deprecated(k8.c_str()); - - std::wstring DirName; - - DirName = u'0' + ((Hash / 100) % 10); - FilePath /= DirName; - DirName = u'0' + ((Hash / 10) % 10); - FilePath /= DirName; - DirName = u'0' + (Hash % 10); - FilePath /= DirName; - - FilePath /= Key; - - auto NativePath = FilePath.native(); - NativePath.append(L".udd"); - - return NativePath; -} - -} // namespace UE - -////////////////////////////////////////////////////////////////////////// - -FileCacheStore::FileCacheStore(const char* RootDir, const char* ReadRootDir) -{ - // Ensure root directory exists - create if it doesn't exist already - - ZEN_INFO("Initializing FileCacheStore at '{}'", std::string_view(RootDir)); - - m_RootDir = RootDir; - - std::error_code ErrorCode; - - std::filesystem::create_directories(m_RootDir, ErrorCode); - - if (ErrorCode) - { - ExtendableStringBuilder<256> Name; - WideToUtf8(m_RootDir.c_str(), Name); - - ZEN_ERROR("Could not open file cache directory '{}' for writing ({})", Name.c_str(), ErrorCode.message()); - - m_IsOk = false; - } - - if (ReadRootDir) - { - m_ReadRootDir = ReadRootDir; - - if (std::filesystem::exists(m_ReadRootDir, ErrorCode)) - { - ZEN_INFO("FileCacheStore will use additional read tree at '{}'", std::string_view(ReadRootDir)); - - m_ReadRootIsValid = true; - } - } -} - -FileCacheStore::~FileCacheStore() -{ -} - -bool -FileCacheStore::Get(std::string_view Key, CacheValue& OutValue) -{ - CAtlFile File; - - std::filesystem::path NativePath; - - HRESULT hRes = E_FAIL; - - if (m_ReadRootDir.empty() == false) - { - NativePath = UE::GenerateDdcPath(Key, m_ReadRootDir); - - hRes = File.Create(NativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); - } - - if (FAILED(hRes)) - { - NativePath = UE::GenerateDdcPath(Key, m_RootDir); - - hRes = File.Create(NativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); - } - - if (FAILED(hRes)) - { - ZEN_DEBUG("GET MISS {}", Key); - - return false; - } - - ULONGLONG FileSize; - File.GetSize(FileSize); - - if (FileSize <= 16) - { - return false; - } - - FileSize -= 16; // CorruptionWrapper trailer - - OutValue.Value = IoBuffer(IoBuffer::File, File.Detach(), 0, FileSize); - - ZEN_DEBUG("GET HIT {}", Key); - - return true; -} - -void -FileCacheStore::Put(std::string_view Key, const CacheValue& Value) -{ - const void* Data = Value.Value.Data(); - size_t Size = Value.Value.Size(); - - UE::CorruptionTrailer Trailer; - Trailer.Initialize(Data, Size); - - std::filesystem::path NativePath = UE::GenerateDdcPath(Key, m_RootDir); - - CAtlTemporaryFile File; - - ZEN_DEBUG("PUT {}", Key); - - HRESULT hRes = File.Create(m_RootDir.c_str()); - - if (SUCCEEDED(hRes)) - { - const uint8_t* WritePointer = reinterpret_cast<const uint8_t*>(Data); - - while (Size) - { - const int MaxChunkSize = 16 * 1024 * 1024; - const int ChunkSize = (int)((Size > MaxChunkSize) ? MaxChunkSize : Size); - - DWORD BytesWritten = 0; - File.Write(WritePointer, ChunkSize, &BytesWritten); - - Size -= BytesWritten; - WritePointer += BytesWritten; - } - - File.Write(&Trailer, sizeof Trailer); - hRes = File.Close(NativePath.c_str()); // This renames the file to its final name - - if (FAILED(hRes)) - { - ZEN_WARN("Failed to rename temp file for key '{}' - deleting temporary file", Key); - - if (!DeleteFile(File.TempFileName())) - { - ZEN_WARN("Temp file for key '{}' could not be deleted - no value persisted", Key); - } - } - } -} - -////////////////////////////////////////////////////////////////////////// - -MemoryCacheStore::MemoryCacheStore() -{ -} - -MemoryCacheStore::~MemoryCacheStore() -{ -} - -bool -MemoryCacheStore::Get(std::string_view InKey, CacheValue& OutValue) -{ - RwLock::SharedLockScope _(m_Lock); - - auto it = m_CacheMap.find(std::string(InKey)); - - if (it == m_CacheMap.end()) - { - return false; - } - else - { - OutValue.Value = it->second; - - return true; - } -} - -void -MemoryCacheStore::Put(std::string_view Key, const CacheValue& Value) -{ - RwLock::ExclusiveLockScope _(m_Lock); - m_CacheMap[std::string(Key)] = Value.Value; -} diff --git a/zenserver/cache/cachestore.h b/zenserver/cache/cachestore.h deleted file mode 100644 index 89c6396b8..000000000 --- a/zenserver/cache/cachestore.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/IoBuffer.h> -#include <zencore/iohash.h> -#include <zencore/thread.h> -#include <zencore/uid.h> -#include <zenstore/cas.h> -#include <compare> -#include <filesystem> -#include <unordered_map> - -namespace zen { - -class WideStringBuilderBase; -class CasStore; - -} // namespace zen - -struct CacheValue -{ - zen::IoBuffer Value; -}; - -/****************************************************************************** - - /$$ /$$/$$ /$$ /$$$$$$ /$$ - | $$ /$$| $$ | $$ /$$__ $$ | $$ - | $$ /$$/| $$ | $$ | $$ \__/ /$$$$$$ /$$$$$$| $$$$$$$ /$$$$$$ - | $$$$$/ | $$ / $$/ | $$ |____ $$/$$_____| $$__ $$/$$__ $$ - | $$ $$ \ $$ $$/ | $$ /$$$$$$| $$ | $$ \ $| $$$$$$$$ - | $$\ $$ \ $$$/ | $$ $$/$$__ $| $$ | $$ | $| $$_____/ - | $$ \ $$ \ $/ | $$$$$$| $$$$$$| $$$$$$| $$ | $| $$$$$$$ - |__/ \__/ \_/ \______/ \_______/\_______|__/ |__/\_______/ - - Basic Key-Value cache. No restrictions on keys, and values are always opaque - binary blobs. - -******************************************************************************/ - -class CacheStore -{ -public: - virtual bool Get(std::string_view Key, CacheValue& OutValue) = 0; - virtual void Put(std::string_view Key, const CacheValue& Value) = 0; -}; - -/** File system based implementation - - Emulates the behaviour of UE4 with regards to file system structure, - and also adds a file corruption trailer to remain compatible with - the file-system based implementation (this should be made configurable) - - */ -class FileCacheStore : public CacheStore -{ -public: - FileCacheStore(const char* RootDir, const char* ReadRootDir = nullptr); - ~FileCacheStore(); - - virtual bool Get(std::string_view Key, CacheValue& OutValue) override; - virtual void Put(std::string_view Key, const CacheValue& Value) override; - -private: - std::filesystem::path m_RootDir; - std::filesystem::path m_ReadRootDir; - bool m_IsOk = true; - bool m_ReadRootIsValid = false; -}; - -class MemoryCacheStore : public CacheStore -{ -public: - MemoryCacheStore(); - ~MemoryCacheStore(); - - virtual bool Get(std::string_view Key, CacheValue& OutValue) override; - virtual void Put(std::string_view Key, const CacheValue& Value) override; - -private: - zen::RwLock m_Lock; - std::unordered_map<std::string, zen::IoBuffer> m_CacheMap; -}; diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index 1671d98a6..29436d840 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -115,7 +115,6 @@ <ClInclude Include="upstream\jupiter.h" /> <ClInclude Include="projectstore.h" /> <ClInclude Include="cache\cacheagent.h" /> - <ClInclude Include="cache\cachestore.h" /> <ClInclude Include="testing\launch.h" /> <ClInclude Include="casstore.h" /> <ClInclude Include="diag\diagsvcs.h" /> @@ -138,7 +137,6 @@ <ClCompile Include="testing\httptest.cpp" /> <ClCompile Include="upstream\jupiter.cpp" /> <ClCompile Include="testing\launch.cpp" /> - <ClCompile Include="cache\cachestore.cpp" /> <ClCompile Include="casstore.cpp" /> <ClCompile Include="experimental\usnjournal.cpp" /> <ClCompile Include="upstream\upstreamcache.cpp" /> diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters index c51a8eb76..6b99ca8d7 100644 --- a/zenserver/zenserver.vcxproj.filters +++ b/zenserver/zenserver.vcxproj.filters @@ -9,9 +9,6 @@ <ClInclude Include="cache\cacheagent.h"> <Filter>cache</Filter> </ClInclude> - <ClInclude Include="cache\cachestore.h"> - <Filter>cache</Filter> - </ClInclude> <ClInclude Include="diag\diagsvcs.h"> <Filter>diag</Filter> </ClInclude> @@ -50,9 +47,6 @@ <ClCompile Include="cache\cacheagent.cpp"> <Filter>cache</Filter> </ClCompile> - <ClCompile Include="cache\cachestore.cpp"> - <Filter>cache</Filter> - </ClCompile> <ClCompile Include="experimental\usnjournal.cpp"> <Filter>experimental</Filter> </ClCompile> |