Add `import-project` and `export-project` (#183)

* Add `import-project` and `export-project` command line parsing

6 files changed, 45 insertions, 19 deletions

diff --git a/zenstore/cas.cpp b/zenstore/cas.cpp
index f8fc41341..fdec78c60 100644
--- a/zenstore/cas.cpp
+++ b/zenstore/cas.cpp
@@ -47,7 +47,7 @@ public:
 	virtual ~CasImpl();
 
 	virtual void				   Initialize(const CidStoreConfiguration& InConfig) override;
-	virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) override;
+	virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, InsertMode Mode) override;
 	virtual IoBuffer			   FindChunk(const IoHash& ChunkHash) override;
 	virtual bool				   ContainsChunk(const IoHash& ChunkHash) override;
 	virtual void				   FilterChunks(HashKeySet& InOutChunks) override;
@@ -191,7 +191,7 @@ CasImpl::UpdateManifest()
 }
 
 CasStore::InsertResult
-CasImpl::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
+CasImpl::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, InsertMode Mode)
 {
 	ZEN_TRACE_CPU("CAS::InsertChunk");
 
@@ -208,7 +208,7 @@ CasImpl::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 		return m_SmallStrategy.InsertChunk(Chunk, ChunkHash);
 	}
 
-	return m_LargeStrategy.InsertChunk(Chunk, ChunkHash);
+	return m_LargeStrategy.InsertChunk(Chunk, ChunkHash, Mode);
 }
 
 IoBuffer
diff --git a/zenstore/cas.h b/zenstore/cas.h
index 2ad160d28..9c48d4707 100644
--- a/zenstore/cas.h
+++ b/zenstore/cas.h
@@ -39,15 +39,21 @@ public:
 		bool New = false;
 	};
 
-	virtual void		 Initialize(const CidStoreConfiguration& Config)	 = 0;
-	virtual InsertResult InsertChunk(IoBuffer Data, const IoHash& ChunkHash) = 0;
-	virtual IoBuffer	 FindChunk(const IoHash& ChunkHash)					 = 0;
-	virtual bool		 ContainsChunk(const IoHash& ChunkHash)				 = 0;
-	virtual void		 FilterChunks(HashKeySet& InOutChunks)				 = 0;
-	virtual void		 Flush()											 = 0;
-	virtual void		 Scrub(ScrubContext& Ctx)							 = 0;
-	virtual void		 GarbageCollect(GcContext& GcCtx)					 = 0;
-	virtual CidStoreSize TotalSize() const									 = 0;
+	enum class InsertMode
+	{
+		kCopyOnly,
+		kMayBeMovedInPlace
+	};
+
+	virtual void		 Initialize(const CidStoreConfiguration& Config)													   = 0;
+	virtual InsertResult InsertChunk(IoBuffer Data, const IoHash& ChunkHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace) = 0;
+	virtual IoBuffer	 FindChunk(const IoHash& ChunkHash)																	   = 0;
+	virtual bool		 ContainsChunk(const IoHash& ChunkHash)																   = 0;
+	virtual void		 FilterChunks(HashKeySet& InOutChunks)																   = 0;
+	virtual void		 Flush()																							   = 0;
+	virtual void		 Scrub(ScrubContext& Ctx)																			   = 0;
+	virtual void		 GarbageCollect(GcContext& GcCtx)																	   = 0;
+	virtual CidStoreSize TotalSize() const																					   = 0;
 
 protected:
 	CidStoreConfiguration m_Config;
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 5a079dbed..8b2797ce9 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -23,14 +23,14 @@ struct CidStore::Impl
 
 	void Initialize(const CidStoreConfiguration& Config) { m_CasStore.Initialize(Config); }
 
-	CidStore::InsertResult AddChunk(const CompressedBuffer& ChunkData)
+	CidStore::InsertResult AddChunk(const CompressedBuffer& ChunkData, CidStore::InsertMode Mode)
 	{
 		const IoHash DecompressedId = IoHash::FromBLAKE3(ChunkData.GetRawHash());
 		IoBuffer	 Payload		= ChunkData.GetCompressed().Flatten().AsIoBuffer();
 
 		Payload.SetContentType(ZenContentType::kCompressedBinary);
 
-		CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, DecompressedId);
+		CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, DecompressedId, static_cast<CasStore::InsertMode>(Mode));
 
 		return {.New = Result.New};
 	}
@@ -78,9 +78,9 @@ CidStore::Initialize(const CidStoreConfiguration& Config)
 }
 
 CidStore::InsertResult
-CidStore::AddChunk(const CompressedBuffer& ChunkData)
+CidStore::AddChunk(const CompressedBuffer& ChunkData, InsertMode Mode)
 {
-	return m_Impl->AddChunk(ChunkData);
+	return m_Impl->AddChunk(ChunkData, Mode);
 }
 
 IoBuffer
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index d0a8ad849..9825f225a 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -124,7 +124,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 }
 
 CasStore::InsertResult
-FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
+FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore::InsertMode Mode)
 {
 	ZEN_ASSERT(m_IsInitialized);
 
@@ -132,6 +132,16 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 	ZEN_ASSERT(Chunk.GetContentType() == ZenContentType::kCompressedBinary);
 #endif
 
+	if (Mode == CasStore::InsertMode::kCopyOnly)
+	{
+		ShardingHelper Name(m_RootDirectory.c_str(), ChunkHash);
+		if (std::filesystem::is_regular_file(Name.ShardedPath.ToPath()))
+		{
+			return {.New = false};
+		}
+		return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash);
+	}
+
 	// File-based chunks have special case handling whereby we move the file into
 	// place in the file store directory, thus avoiding unnecessary copying
 
@@ -153,6 +163,8 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 
 			if (!Success)
 			{
+				// TODO: We should provide information to this function to tell it if the payload is temporary or not and if we are allowed
+				// to delete it.
 				ZEN_WARN("Failed to flag temporary payload file '{}' for deletion: '{}'",
 						 PathFromHandle(ChunkFileHandle),
 						 GetLastErrorAsString());
@@ -294,6 +306,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 				 ChunkHash);
 
 		DeletePayloadFileOnClose();
+
 #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 		std::filesystem::path SourcePath = PathFromHandle(FileRef.FileHandle);
 		std::filesystem::path DestPath	 = Name.ShardedPath.c_str();
diff --git a/zenstore/filecas.h b/zenstore/filecas.h
index f14e5d057..de79b8b81 100644
--- a/zenstore/filecas.h
+++ b/zenstore/filecas.h
@@ -34,7 +34,9 @@ struct FileCasStrategy final : public GcStorage
 
 	void				   Initialize(const std::filesystem::path& RootDirectory, bool IsNewStore);
 	CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
-	CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash);
+	CasStore::InsertResult InsertChunk(IoBuffer				Chunk,
+									   const IoHash&		ChunkHash,
+									   CasStore::InsertMode Mode = CasStore::InsertMode::kMayBeMovedInPlace);
 	IoBuffer			   FindChunk(const IoHash& ChunkHash);
 	bool				   HaveChunk(const IoHash& ChunkHash);
 	void				   FilterChunks(HashKeySet& InOutChunks);
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
index 21e3c3160..e8984a83d 100644
--- a/zenstore/include/zenstore/cidstore.h
+++ b/zenstore/include/zenstore/cidstore.h
@@ -63,9 +63,14 @@ public:
 	{
 		bool New = false;
 	};
+	enum class InsertMode
+	{
+		kCopyOnly,
+		kMayBeMovedInPlace
+	};
 
 	void		 Initialize(const CidStoreConfiguration& Config);
-	InsertResult AddChunk(const CompressedBuffer& ChunkData);
+	InsertResult AddChunk(const CompressedBuffer& ChunkData, InsertMode Mode = InsertMode::kMayBeMovedInPlace);
 	IoBuffer	 FindChunkByCid(const IoHash& DecompressedId);
 	bool		 ContainsChunk(const IoHash& DecompressedId);
 	void		 FilterChunks(HashKeySet& InOutChunks);