1 files changed, 462 insertions, 0 deletions
diff --git a/public/vpklib/packedstore.h b/public/vpklib/packedstore.h
new file mode 100644
index 0000000..1d5c40c
--- /dev/null
+++ b/public/vpklib/packedstore.h
@@ -0,0 +1,462 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//===========================================================================//
+
+#ifndef PACKEDSTORE_H
+#define PACKEDSTORE_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+
+#include <tier0/platform.h>
+#include <tier0/threadtools.h>
+#include <tier0/tslist.h>
+#include <tier2/tier2.h>
+
+#include "filesystem.h"
+#include "tier1/utlintrusivelist.h"
+#include "tier1/utlvector.h"
+#include "tier1/utllinkedlist.h"
+#include "tier1/UtlSortVector.h"
+#include "tier1/utlmap.h"
+#include "tier1/checksum_md5.h"
+
+#define VPK_ENABLE_SIGNING
+
+const int k_nVPKDefaultChunkSize = 200 * 1024 * 1024;
+
+class CPackedStore;
+
+
+struct ChunkHashFraction_t
+{
+	int m_nPackFileNumber;
+	int m_nFileFraction;
+	int m_cbChunkLen;
+	MD5Value_t m_md5contents;
+};
+
+class ChunkHashFractionLess_t
+{
+public:
+	bool Less( const ChunkHashFraction_t& lhs, const ChunkHashFraction_t& rhs, void *pContext )
+	{
+		if ( lhs.m_nPackFileNumber < rhs.m_nPackFileNumber )
+			return true;
+		if ( lhs.m_nPackFileNumber > rhs.m_nPackFileNumber )
+			return false;
+
+		if ( lhs.m_nFileFraction < rhs.m_nFileFraction )
+			return true;
+		if ( lhs.m_nFileFraction > rhs.m_nFileFraction )
+			return false;
+		return false;
+	}
+};
+
+class CPackedStoreFileHandle
+{
+public:
+	int m_nFileNumber;
+	int m_nFileOffset;
+	int m_nFileSize;
+	int m_nCurrentFileOffset;
+	void const *m_pMetaData;
+	uint16 m_nMetaDataSize;
+	CPackedStore *m_pOwner;
+	struct CFileHeaderFixedData *m_pHeaderData;
+	uint8 *m_pDirFileNamePtr;								// pointer to basename in dir block
+
+	FORCEINLINE operator bool( void ) const
+	{
+		return ( m_nFileNumber != -1 );
+	}
+
+	FORCEINLINE int Read( void *pOutData, int nNumBytes );
+
+	CPackedStoreFileHandle( void )
+	{
+		m_nFileNumber = -1;
+	}
+
+	int Seek( int nOffset, int nWhence )
+	{
+		switch( nWhence )
+		{
+			case SEEK_CUR:
+				nOffset = m_nFileOffset + nOffset ;
+				break;
+
+			case SEEK_END:
+				nOffset = m_nFileSize + nOffset;
+				break;
+		}
+		m_nCurrentFileOffset = MAX( 0, MIN( m_nFileSize, nOffset ) );
+		return m_nCurrentFileOffset;
+	}
+
+	int Tell( void ) const
+	{
+		return m_nCurrentFileOffset;
+	}
+
+	uint32 GetFileCRCFromHeaderData() const
+	{
+		uint32 *pCRC = (uint32 *)m_pHeaderData;
+		return *pCRC;
+	}
+
+	FORCEINLINE void GetPackFileName( char *pchFileNameOut, int cchFileNameOut );
+
+};
+
+#define MAX_ARCHIVE_FILES_TO_KEEP_OPEN_AT_ONCE 512
+
+#define PACKEDFILE_EXT_HASH_SIZE 15
+
+
+#ifdef _WIN32
+typedef HANDLE PackDataFileHandle_t;
+#else
+typedef FileHandle_t PackDataFileHandle_t;
+#endif
+
+struct FileHandleTracker_t
+{
+	int m_nFileNumber;
+	PackDataFileHandle_t m_hFileHandle;
+	int m_nCurOfs;
+	CThreadFastMutex m_Mutex;
+
+	FileHandleTracker_t( void )
+	{
+		m_nFileNumber = -1;
+	}
+};
+
+enum ePackedStoreAddResultCode
+{
+	EPADD_NEWFILE,											// the file was added and is new
+	EPADD_ADDSAMEFILE,										// the file was already present, and the contents are the same as what you passed.
+	EPADD_UPDATEFILE,										// the file was alreayd present and its contents have been updated
+	EPADD_ERROR,											// some error has resulted
+};
+
+// Describe a file inside of a VPK file.  Is not memory efficient; only used for interface
+// purposes and during file building
+struct VPKContentFileInfo_t
+{
+	CUtlString m_sName;
+	int m_idxChunk;
+	uint32 m_iTotalSize;
+	uint32 m_iOffsetInChunk;
+	uint32 m_iPreloadSize;
+	const void *m_pPreloadData;
+	//MD5Value_t m_md5Source; // source content before munging & release optimization.  Used for incremental builds
+	uint32 m_crc; // CRC of actual file contents
+
+	/// Size of the data in the chunk file.  (Excludes the preload data size)
+	uint32 GetSizeInChunkFile() const
+	{
+		Assert( m_iTotalSize >= m_iPreloadSize );
+		return m_iTotalSize - m_iPreloadSize;
+	}
+
+	VPKContentFileInfo_t()
+	{
+		m_idxChunk = -1;
+		m_iTotalSize = 0;
+		m_iOffsetInChunk = 0;
+		m_iPreloadSize = 0;
+		m_crc = 0;
+		m_pPreloadData = NULL;
+		//memset( m_md5Source.bits, 0, sizeof( m_md5Source.bits ) );
+	}
+};
+
+
+// a 1MB chunk of cached VPK data
+// For CPackedStoreReadCache
+struct CachedVPKRead_t
+{
+	CachedVPKRead_t()
+	{
+		m_nPackFileNumber = 0;
+		m_nFileFraction = 0;
+		m_pubBuffer = NULL;
+		m_cubBuffer = 0;
+		m_idxLRU = -1;
+		m_hMD5RequestHandle= 0;
+		m_cFailedHashes = 0;
+	}
+	int m_nPackFileNumber;	// identifier
+	int m_nFileFraction;	// identifier
+	uint8 *m_pubBuffer;		// data
+	int m_cubBuffer;		// data
+	int m_idxLRU;			// bookkeeping
+	int m_hMD5RequestHandle;// bookkeeping
+	int m_cFailedHashes;	// did the MD5 match what it was supposed to?
+	MD5Value_t m_md5Value;
+
+	static bool Less( const CachedVPKRead_t& lhs, const CachedVPKRead_t& rhs )
+	{
+		if ( lhs.m_nPackFileNumber < rhs.m_nPackFileNumber )
+			return true;
+		if ( lhs.m_nPackFileNumber > rhs.m_nPackFileNumber )
+			return false;
+		if ( lhs.m_nFileFraction < rhs.m_nFileFraction )
+			return true;
+		if ( lhs.m_nFileFraction > rhs.m_nFileFraction )
+			return false;
+		return false;
+	}
+
+};
+
+
+// Read the VPK file in 1MB chunks
+// and we hang on to those chunks so we can serve other reads out of the cache
+// This sounds great, but is only of secondary importance.
+// The primary reason we do this is so that the FileTracker can calculate the 
+// MD5 of the 1MB chunks asynchronously in another thread - while we hold
+// the chunk in cache - making the MD5 calculation "free"
+class CPackedStoreReadCache
+{
+public:
+	CPackedStoreReadCache( IBaseFileSystem *pFS );
+
+	bool ReadCacheLine( FileHandleTracker_t &fHandle, CachedVPKRead_t &cachedVPKRead );
+	bool BCanSatisfyFromReadCache( uint8 *pOutData, CPackedStoreFileHandle &handle, FileHandleTracker_t &fHandle, int nDesiredPos, int nNumBytes, int &nRead );
+	bool BCanSatisfyFromReadCacheInternal( uint8 *pOutData, CPackedStoreFileHandle &handle, FileHandleTracker_t &fHandle, int nDesiredPos, int nNumBytes, int &nRead );
+	bool CheckMd5Result( CachedVPKRead_t &cachedVPKRead );
+	int FindBufferToUse();
+	void RetryBadCacheLine( CachedVPKRead_t &cachedVPKRead );
+	void RetryAllBadCacheLines();
+
+
+	// cache 64 MB total
+	static const int k_nCacheBuffersToKeep = 4;
+	static const int k_cubCacheBufferSize = 0x00100000; // 1MB
+	static const int k_nCacheBufferMask = 0x7FF00000;
+
+	CThreadRWLock m_rwlock;
+	CUtlRBTree<CachedVPKRead_t> m_treeCachedVPKRead; // all the reads we have done
+
+	CTSQueue<CachedVPKRead_t> m_queueCachedVPKReadsRetry; // all the reads that have failed
+	CUtlLinkedList<CachedVPKRead_t> m_listCachedVPKReadsFailed; // all the reads that have failed
+
+	// current items in the cache
+	int m_cItemsInCache;
+	int m_rgCurrentCacheIndex[k_nCacheBuffersToKeep];
+	CInterlockedUInt m_rgLastUsedTime[k_nCacheBuffersToKeep];
+
+	CPackedStore *m_pPackedStore;
+	IBaseFileSystem *m_pFileSystem;
+	IThreadedFileMD5Processor *m_pFileTracker;
+	// stats
+	int m_cubReadFromCache;
+	int m_cReadFromCache;
+	int m_cDiscardsFromCache;
+	int m_cAddedToCache;
+	int m_cCacheMiss;
+	int m_cubCacheMiss;
+	int m_cFileErrors;
+	int m_cFileErrorsCorrected;
+	int m_cFileResultsDifferent;
+};
+
+class CPackedStore
+{
+public:
+	CPackedStore( char const *pFileBasename, char *pszFName, IBaseFileSystem *pFS, bool bOpenForWrite = false );
+
+	void RegisterFileTracker( IThreadedFileMD5Processor *pFileTracker ) { m_pFileTracker = pFileTracker; m_PackedStoreReadCache.m_pFileTracker = pFileTracker; }
+
+	CPackedStoreFileHandle OpenFile( char const *pFile );
+	CPackedStoreFileHandle GetHandleForHashingFiles();
+
+	/// Add/update the given file to the directory.  Does not write any chunk files
+	void AddFileToDirectory( const VPKContentFileInfo_t &info );
+
+	/// Remove the specified file from the directory.  Returns true if removed, false if not found
+	bool RemoveFileFromDirectory( const char *pszName );
+
+	/// Add file, writing file data to the end
+	/// of the current chunk
+	ePackedStoreAddResultCode AddFile( char const *pFile, uint16 nMetaDataSize, const void *pFileData, uint32 nFullFileSize, bool bMultiChunk, uint32 const *pCrcToUse = NULL );
+
+	// write out the file directory
+	void Write( void );
+
+	int ReadData( CPackedStoreFileHandle &handle, void *pOutData, int nNumBytes );
+
+	~CPackedStore( void );
+
+	FORCEINLINE void *DirectoryData( void )
+	{
+		return m_DirectoryData.Base();
+	}
+
+	// Get a list of all the files in the zip You are responsible for freeing the contents of
+	// outFilenames (call outFilenames.PurgeAndDeleteElements).
+	int GetFileList( CUtlStringList &outFilenames, bool bFormattedOutput, bool bSortedOutput );
+
+	// Get a list of all files that match the given wildcard string
+	int GetFileList( const char *pWildCard, CUtlStringList &outFilenames, bool bFormattedOutput, bool bSortedOutput );
+	
+	/// Get a list of all files that match the given wildcard string, fetching all the details
+	/// at once
+	void GetFileList( const char *pWildcard, CUtlVector<VPKContentFileInfo_t> &outVecResults );
+
+	// Get a list of all directories of the given wildcard
+	int GetFileAndDirLists( const char *pWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput );
+	int GetFileAndDirLists( CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput );
+
+	bool IsEmpty( void ) const;
+
+	/// Hash metadata and chunk files
+	void HashEverything();
+
+	/// Hash all chunk files.  Don't forget to rehash the metadata afterwords!
+	void HashAllChunkFiles();
+
+	/// Hash all the metadata.  (Everything that's not in the chunk files)
+	void HashMetadata();
+
+	/// Re-hash a single chunk file.  Don't forget to rehash the metadata afterwords!
+	void HashChunkFile( int iChunkFileIndex );
+
+	bool HashEntirePackFile( CPackedStoreFileHandle &handle, int64 &nFileSize, int nFileFraction, int nFractionSize, FileHash_t &fileHash );
+	void ComputeDirectoryHash( MD5Value_t &md5Directory );
+	void ComputeChunkHash( MD5Value_t &md5ChunkHashes );
+	MD5Value_t &GetDirFileMD5Value() { return m_TotalFileMD5; }
+	bool BTestDirectoryHash();
+	bool BTestMasterChunkHash();
+	CUtlSortVector<ChunkHashFraction_t, ChunkHashFractionLess_t > &AccessPackFileHashes() { return m_vecChunkHashFraction; }
+	bool FindFileHashFraction( int nPackFileNumber, int nFileFraction, ChunkHashFraction_t &chunkFileHashFraction );
+	void GetPackFileLoadErrorSummary( CUtlString &sErrors );
+
+	void GetPackFileName( CPackedStoreFileHandle &handle, char *pchFileNameOut, int cchFileNameOut ) const;
+	void GetDataFileName( char *pchFileNameOut, int cchFileNameOut, int nFileNumber ) const;
+
+	char const *BaseName( void )
+	{
+		return m_pszFileBaseName;
+	}
+
+	char const *FullPathName( void )
+	{
+		return m_pszFullPathName;
+	}
+
+	void SetWriteChunkSize( int nWriteChunkSize )
+	{
+		m_nWriteChunkSize = nWriteChunkSize;
+	}
+
+	int GetWriteChunkSize() const { return m_nWriteChunkSize; }
+
+	int GetHighestChunkFileIndex() { return m_nHighestChunkFileIndex; }
+
+	void DiscardChunkHashes( int iChunkFileIndex );
+
+	const CUtlVector<uint8> &GetSignaturePublicKey() const { return m_SignaturePublicKey; }
+	const CUtlVector<uint8> &GetSignature() const { return m_Signature; }
+
+#ifdef VPK_ENABLE_SIGNING
+	enum ESignatureCheckResult
+	{
+		eSignatureCheckResult_NotSigned,
+		eSignatureCheckResult_WrongKey,
+		eSignatureCheckResult_Failed, // IO error, etc
+		eSignatureCheckResult_InvalidSignature,
+		eSignatureCheckResult_ValidSignature,
+	};
+	ESignatureCheckResult CheckSignature( int nSignatureSize, const void *pSignature ) const;
+
+	void SetKeysForSigning( int nPrivateKeySize, const void *pPrivateKeyData, int nPublicKeySize, const void *pPublicKeyData );
+#endif
+
+	void SetUseDirFile() { m_bUseDirFile = true; }
+
+	int m_PackFileID;
+private:
+	char m_pszFileBaseName[MAX_PATH];
+	char m_pszFullPathName[MAX_PATH];
+	int m_nDirectoryDataSize;
+	int m_nWriteChunkSize;
+	bool m_bUseDirFile;
+
+	IBaseFileSystem *m_pFileSystem;
+	IThreadedFileMD5Processor *m_pFileTracker;
+	CThreadFastMutex m_Mutex;
+	
+	CPackedStoreReadCache m_PackedStoreReadCache;
+
+	CUtlIntrusiveList<class CFileExtensionData> m_pExtensionData[PACKEDFILE_EXT_HASH_SIZE];
+
+	CUtlVector<uint8> m_DirectoryData;
+	CUtlBlockVector<uint8> m_EmbeddedChunkData;
+
+	CUtlSortVector<ChunkHashFraction_t, ChunkHashFractionLess_t > m_vecChunkHashFraction;
+	bool BFileContainedHashes() { return m_vecChunkHashFraction.Count() > 0; }
+	// these are valid if BFileContainedHashes() is true
+	MD5Value_t m_DirectoryMD5;
+	MD5Value_t m_ChunkHashesMD5;
+	MD5Value_t m_TotalFileMD5;
+
+	int m_nHighestChunkFileIndex;
+
+	/// The private key that will be used to sign the directory file.
+	/// This will be empty for unsigned VPK's, or if we don't know the
+	/// private key.
+	CUtlVector<uint8> m_SignaturePrivateKey;
+
+	/// The public key in the VPK.
+	CUtlVector<uint8> m_SignaturePublicKey;
+
+	/// The signature that was read / computed
+	CUtlVector<uint8> m_Signature;
+
+	/// The number of bytes in the dir file that were signed
+	uint32 m_nSizeOfSignedData;
+
+	FileHandleTracker_t m_FileHandles[MAX_ARCHIVE_FILES_TO_KEEP_OPEN_AT_ONCE];
+	
+	void Init( void );
+
+	struct CFileHeaderFixedData *FindFileEntry( 
+		char const *pDirname, char const *pBaseName, char const *pExtension,
+		uint8 **pExtBaseOut = NULL, uint8 **pNameBaseOut = NULL );
+
+	void BuildHashTables( void );
+
+	FileHandleTracker_t &GetFileHandle( int nFileNumber );
+
+	void CloseWriteHandle( void );
+
+	// For cache-ing directory and contents data
+	CUtlStringList m_directoryList; // The index of this list of directories...
+	CUtlMap<int, CUtlStringList*> m_dirContents; // ...is the key to this map of filenames
+	void BuildFindFirstCache();
+
+	bool InternalRemoveFileFromDirectory( const char *pszName );
+
+	friend class CPackedStoreReadCache;
+};
+
+FORCEINLINE int CPackedStoreFileHandle::Read( void *pOutData, int nNumBytes )
+{
+	return m_pOwner->ReadData( *this, pOutData, nNumBytes );
+}
+
+FORCEINLINE void CPackedStoreFileHandle::GetPackFileName( char *pchFileNameOut, int cchFileNameOut )
+{
+	m_pOwner->GetPackFileName( *this, pchFileNameOut, cchFileNameOut );
+}
+
+
+#endif // packedtsore_h