From c7e22a4ef1cce7103b9afbeec487461cb32f8dbe Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Fri, 17 Jun 2022 07:06:21 -0700
Subject: Make cas storage an hidden implementation detail of CidStore (#130)

- Bumped ZEN_SCHEMA_VERSION
- CasStore no longer a public API, it is hidden behind CidStore
- Moved cas.h from public header folder
- CidStore no longer maps from Cid -> Cas, we store entries in Cas under RawHash
- CasStore now decompresses data to validate content (matching against RawHash)
- CasChunkSet renames to HashKeySet and put in separate header/cpp file
- Disabled "Chunk" command for now as it relied on CAS being exposed as a service
- Changed CAS http service to Cid http server
- Moved "Run" command completely inside ZEN_WITH_EXEC_SERVICES define
- Removed "cas.basic" test
- Uncommented ".exec.basic" test and added return-skip at start of test
- Moved ScrubContext to separate header file
- Renamed CasGC to GcManager
- Cleaned up configuration passing in cas store classes
- Removed CAS stuff from GcContext and clarified naming in class
- Remove migration code
---
 zenserver/compute/function.cpp | 33 +++++++++++----------------------
 1 file changed, 11 insertions(+), 22 deletions(-)

(limited to 'zenserver/compute/function.cpp')

diff --git a/zenserver/compute/function.cpp b/zenserver/compute/function.cpp
index 171c67a6e..d7316ac64 100644
--- a/zenserver/compute/function.cpp
+++ b/zenserver/compute/function.cpp
@@ -17,7 +17,6 @@
 #	include <zencore/iobuffer.h>
 #	include <zencore/iohash.h>
 #	include <zencore/scopeguard.h>
-#	include <zenstore/cas.h>
 #	include <zenstore/cidstore.h>
 
 #	include <span>
@@ -26,25 +25,22 @@ using namespace std::literals;
 
 namespace zen {
 
-HttpFunctionService::HttpFunctionService(CasStore&						Store,
-										 CidStore&						InCidStore,
+HttpFunctionService::HttpFunctionService(CidStore&						InCidStore,
 										 const CloudCacheClientOptions& ComputeOptions,
 										 const CloudCacheClientOptions& StorageOptions,
 										 const UpstreamAuthConfig&		ComputeAuthConfig,
 										 const UpstreamAuthConfig&		StorageAuthConfig,
 										 AuthMgr&						Mgr)
 : m_Log(logging::Get("apply"))
-, m_CasStore(Store)
 , m_CidStore(InCidStore)
 {
-	m_UpstreamApply = UpstreamApply::Create({}, m_CasStore, m_CidStore);
+	m_UpstreamApply = UpstreamApply::Create({}, m_CidStore);
 
 	InitializeThread = std::thread{[this, ComputeOptions, StorageOptions, ComputeAuthConfig, StorageAuthConfig, &Mgr] {
 		auto HordeUpstreamEndpoint = UpstreamApplyEndpoint::CreateHordeEndpoint(ComputeOptions,
 																				ComputeAuthConfig,
 																				StorageOptions,
 																				StorageAuthConfig,
-																				m_CasStore,
 																				m_CidStore,
 																				Mgr);
 		m_UpstreamApply->RegisterEndpoint(std::move(HordeUpstreamEndpoint));
@@ -99,18 +95,18 @@ HttpFunctionService::HttpFunctionService(CasStore&						Store,
 
 									// Determine which pieces are missing and need to be transmitted to populate CAS
 
-									CasChunkSet ChunkSet;
+									HashKeySet ChunkSet;
 
 									FunctionSpec.IterateAttachments([&](CbFieldView Field) {
 										const IoHash Hash = Field.AsHash();
-										ChunkSet.AddChunkToSet(Hash);
+										ChunkSet.AddHashToSet(Hash);
 									});
 
 									// Note that we store executables uncompressed to make it
 									// more straightforward and efficient to materialize them, hence
 									// the CAS lookup here instead of CID for the input payloads
 
-									m_CasStore.FilterChunks(ChunkSet);
+									m_CidStore.FilterChunks(ChunkSet);
 
 									if (ChunkSet.IsEmpty())
 									{
@@ -127,7 +123,7 @@ HttpFunctionService::HttpFunctionService(CasStore&						Store,
 										CbObjectWriter ResponseWriter;
 										ResponseWriter.BeginArray("need");
 
-										ChunkSet.IterateChunks([&](const IoHash& Hash) {
+										ChunkSet.IterateHashes([&](const IoHash& Hash) {
 											ZEN_DEBUG("worker {}: need chunk {}", WorkerId, Hash);
 
 											ResponseWriter.AddHash(Hash);
@@ -159,25 +155,18 @@ HttpFunctionService::HttpFunctionService(CasStore&						Store,
 									{
 										ZEN_ASSERT(Attachment.IsCompressedBinary());
 
-										const IoHash	 DataHash		  = Attachment.GetHash();
-										CompressedBuffer DataView		  = Attachment.AsCompressedBinary();
-										SharedBuffer	 Decompressed	  = DataView.Decompress();
-										const uint64_t	 DecompressedSize = DataView.GetRawSize();
+										const IoHash	 DataHash = Attachment.GetHash();
+										CompressedBuffer Buffer	  = Attachment.AsCompressedBinary();
 
 										ZEN_UNUSED(DataHash);
-
-										TotalAttachmentBytes += DecompressedSize;
+										TotalAttachmentBytes += Buffer.GetCompressedSize();
 										++AttachmentCount;
 
-										// Note that we store executables uncompressed to make it
-										// more straightforward and efficient to materialize them
-
-										const CasStore::InsertResult InsertResult =
-											m_CasStore.InsertChunk(Decompressed.AsIoBuffer(), IoHash::FromBLAKE3(DataView.GetRawHash()));
+										const CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Buffer);
 
 										if (InsertResult.New)
 										{
-											TotalNewBytes += DecompressedSize;
+											TotalNewBytes += Buffer.GetCompressedSize();
 											++NewAttachmentCount;
 										}
 									}
-- 
cgit v1.2.3