From c7e22a4ef1cce7103b9afbeec487461cb32f8dbe Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 17 Jun 2022 07:06:21 -0700 Subject: Make cas storage an hidden implementation detail of CidStore (#130) - Bumped ZEN_SCHEMA_VERSION - CasStore no longer a public API, it is hidden behind CidStore - Moved cas.h from public header folder - CidStore no longer maps from Cid -> Cas, we store entries in Cas under RawHash - CasStore now decompresses data to validate content (matching against RawHash) - CasChunkSet renames to HashKeySet and put in separate header/cpp file - Disabled "Chunk" command for now as it relied on CAS being exposed as a service - Changed CAS http service to Cid http server - Moved "Run" command completely inside ZEN_WITH_EXEC_SERVICES define - Removed "cas.basic" test - Uncommented ".exec.basic" test and added return-skip at start of test - Moved ScrubContext to separate header file - Renamed CasGC to GcManager - Cleaned up configuration passing in cas store classes - Removed CAS stuff from GcContext and clarified naming in class - Remove migration code --- zenserver/testing/launch.cpp | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'zenserver/testing/launch.cpp') diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp index 1236e6adb..0e46fff94 100644 --- a/zenserver/testing/launch.cpp +++ b/zenserver/testing/launch.cpp @@ -6,13 +6,14 @@ # include # include +# include # include # include # include # include # include # include -# include +# include ZEN_THIRD_PARTY_INCLUDES_START # include @@ -322,9 +323,9 @@ SandboxedJob::SpawnJob(std::filesystem::path ExePath) //////////////////////////////////////////////////////////////////////////////// -HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::path& SandboxBaseDir) +HttpLaunchService::HttpLaunchService(CidStore& Store, const std::filesystem::path& SandboxBaseDir) : m_Log(logging::Get("exec")) -, m_CasStore(Store) +, m_CidStore(Store) , m_SandboxPath(SandboxBaseDir) { m_Router.AddPattern("job", "([[:digit:]]+)"); @@ -402,7 +403,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat const IoHash FileHash = Ob["hash"sv].AsHash(); - if (!m_CasStore.FindChunk(FileHash)) + if (!m_CidStore.FindChunkByCid(FileHash)) { ZEN_DEBUG("NEED: {} {} {}", FileHash, Ob["file"sv].AsString(), Ob["size"sv].AsUInt64()); @@ -465,7 +466,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat const IoHash FileHash = Ob["hash"sv].AsHash(); uint64_t FileSize = Ob["size"sv].AsUInt64(); - if (IoBuffer Chunk = m_CasStore.FindChunk(FileHash); !Chunk) + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(FileHash); !Chunk) { ZEN_DEBUG("MISSING: {} {} {}", FileHash, FileName, FileSize); AllOk = false; @@ -476,9 +477,18 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat { std::filesystem::path FullPath = SandboxDir / FileName; - const IoBuffer* Chunks[] = {&Chunk}; - - zen::WriteFile(FullPath, Chunks, 1); + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk)); + CompositeBuffer CompositeBuffer = Compressed.DecompressToComposite(); + std::span Segments = CompositeBuffer.GetSegments(); + std::vector Chunks(Segments.size()); + std::vector ChunkPtrs(Segments.size()); + for (size_t Index = 0; Index < Segments.size(); ++Index) + { + Chunks[Index] = std::move(Segments[Index].AsIoBuffer()); + ChunkPtrs[Index] = &Chunks[Index]; + } + + zen::WriteFile(FullPath, ChunkPtrs.data(), ChunkPtrs.size()); } } -- cgit v1.2.3