1HEADmaster

1 files changed, 593 insertions, 0 deletions

diff --git a/utils/xbox/MakeGameData/MakeZip.cpp b/utils/xbox/MakeGameData/MakeZip.cpp
new file mode 100644
index 0000000..69950b0
--- /dev/null
+++ b/utils/xbox/MakeGameData/MakeZip.cpp
@@ -0,0 +1,593 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "MakeGameData.h"
+
+static CUtlSymbolTable g_CriticalPreloadTable( 0, 32, true );
+
+//-----------------------------------------------------------------------------
+// Purpose: Compute preload data by file type. Calls into appropriate libraries
+//			to get the preload info. Libraries would use filename to generate
+//			the preload if there is a compilation step, otherwise the file buffer
+//			is a buffer loaded filename.
+//-----------------------------------------------------------------------------
+static bool GetPreloadBuffer( const char *pFilename, CUtlBuffer &fileBuffer, CUtlBuffer &preloadBuffer )
+{
+	char fileExtension[MAX_PATH];
+	Q_ExtractFileExtension( pFilename, fileExtension, sizeof( fileExtension ) );
+
+	// adding an entire file IS ONLY for files that are expected to be read by the game as a single read
+	// NOT for files that have any seek pattern
+	bool bAddEntireFile = false;
+
+	// trivial small files, always add
+	if ( !Q_stricmp( fileExtension, "txt" ) ||
+		!Q_stricmp( fileExtension, "dat" ) ||
+		!Q_stricmp( fileExtension, "lst" ) ||
+		!Q_stricmp( fileExtension, "res" ) ||
+		!Q_stricmp( fileExtension, "vmt" ) ||
+		!Q_stricmp( fileExtension, "cfg" ) ||
+		!Q_stricmp( fileExtension, "bnf" ) ||
+		!Q_stricmp( fileExtension, "rc" ) ||
+		!Q_stricmp( fileExtension, "vbf" ) ||
+		!Q_stricmp( fileExtension, "vfe" ) ||
+		!Q_stricmp( fileExtension, "pcf" ) ||
+		!Q_stricmp( fileExtension, "inf" ) )
+	{
+		bAddEntireFile = true;
+	}
+
+	// critical resources get blindly added to the preload
+	if ( !bAddEntireFile && ( g_CriticalPreloadTable.Find( pFilename ) != UTL_INVAL_SYMBOL ) )
+	{
+		bAddEntireFile = true;
+	}
+
+	if ( bAddEntireFile && LZMA_IsCompressed( (unsigned char *)fileBuffer.Base() ) )
+	{
+		// sorry, not allowed to add entirely to preload if already compressed
+		// breaks the run-time filesystem due to inability to deliver file as-is
+		bAddEntireFile = false;
+	}
+
+	if ( bAddEntireFile )
+	{
+		if ( fileBuffer.TellMaxPut() >= 1*1024 )
+		{
+			// only compress preload files of reasonable size
+			unsigned int compressedSize = 0;
+			unsigned char *pCompressedOutput = LZMA_OpportunisticCompress( (unsigned char *)fileBuffer.Base(),
+			                                                               fileBuffer.TellMaxPut(), &compressedSize );
+			if ( pCompressedOutput )
+			{
+				// add as compressed
+				preloadBuffer.EnsureCapacity( compressedSize );
+				preloadBuffer.Put( pCompressedOutput, compressedSize );
+				free( pCompressedOutput );
+				return true;
+			}
+		}
+
+		// add entire file to preload section
+		preloadBuffer.EnsureCapacity( fileBuffer.TellMaxPut() );
+		preloadBuffer.Put( fileBuffer.Base(), fileBuffer.TellMaxPut() );
+		return true;
+	}
+
+	// Each library will fetch the optional preload data into caller's buffer
+	if ( !Q_stricmp( fileExtension, "wav" ) )
+	{
+		return GetPreloadData_WAV( pFilename, fileBuffer, preloadBuffer );
+	}
+	else if ( !Q_stricmp( fileExtension, "vtf" ) )
+	{
+		return GetPreloadData_VTF( pFilename, fileBuffer, preloadBuffer );
+	}
+	else if ( !Q_stricmp( fileExtension, "vcs" ) )
+	{
+		return GetPreloadData_VCS( pFilename, fileBuffer, preloadBuffer );
+	}
+	else if ( !Q_stricmp( fileExtension, "vhv" ) )
+	{
+		return GetPreloadData_VHV( pFilename, fileBuffer, preloadBuffer );
+	}
+	else if ( !Q_stricmp( fileExtension, "vtx" ) )
+	{
+		return GetPreloadData_VTX( pFilename, fileBuffer, preloadBuffer );
+	}
+	else if ( !Q_stricmp( fileExtension, "vvd" ) )
+	{
+		return GetPreloadData_VVD( pFilename, fileBuffer, preloadBuffer );
+	}
+
+	// others...
+	return false;
+}
+
+void SetupCriticalPreloadScript( const char *pModPath )
+{
+	characterset_t breakSet;
+	CharacterSetBuild( &breakSet, "" );
+
+	// purge any prior entries
+	g_CriticalPreloadTable.RemoveAll();
+
+	// populate table
+	char szCriticaList[MAX_PATH];
+	char szToken[MAX_PATH];
+	V_ComposeFileName( pModPath, "scripts\\preload_xbox.xsc", szCriticaList, sizeof( szCriticaList ) );
+	CUtlBuffer criticalListBuffer;
+	if ( ReadFileToBuffer( szCriticaList, criticalListBuffer, true, true ) )
+	{
+		for ( ;; )
+		{
+			int nTokenSize = criticalListBuffer.ParseToken( &breakSet, szToken, sizeof( szToken ) );
+			if ( nTokenSize <= 0 )
+			{
+				break;
+			}
+		
+			V_strlower( szToken );
+			V_FixSlashes( szToken, CORRECT_PATH_SEPARATOR );
+			if ( UTL_INVAL_SYMBOL == g_CriticalPreloadTable.Find( szToken ) )
+			{
+				g_CriticalPreloadTable.AddString( szToken );
+			}
+		}
+	}
+}
+
+CXZipTool::CXZipTool()
+{
+	m_pZip = NULL;
+	m_hPreloadFile = INVALID_HANDLE_VALUE;
+	m_hOutputZipFile = INVALID_HANDLE_VALUE;
+	m_PreloadFilename[0] = '\0';
+}
+
+CXZipTool::~CXZipTool()
+{
+	Reset();
+}
+
+void CXZipTool::Reset()
+{
+	if ( m_hOutputZipFile != INVALID_HANDLE_VALUE )
+	{
+		CloseHandle( m_hOutputZipFile );
+		m_hOutputZipFile = INVALID_HANDLE_VALUE;
+	}
+
+	if ( m_hPreloadFile != INVALID_HANDLE_VALUE )
+	{
+		CloseHandle( m_hPreloadFile );
+		m_hPreloadFile = INVALID_HANDLE_VALUE;
+	}
+
+	if ( m_PreloadFilename[0] )
+	{
+		DeleteFile( m_PreloadFilename );
+		m_PreloadFilename[0] = '\0';
+	}
+
+	if ( m_pZip )
+	{
+		IZip::ReleaseZip( m_pZip );
+		m_pZip = NULL;
+	}
+
+	m_ZipPreloadDirectoryEntries.Purge();
+	m_ZipCRCList.Purge();
+	m_ZipPreloadRemapEntries.Purge();
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: Add a file buffer to the zip
+//-----------------------------------------------------------------------------
+bool CXZipTool::AddBuffer( const char *pFilename, CUtlBuffer &fileBuffer, bool bDoPreload )
+{
+	if ( !m_pZip )
+	{
+		return false;
+	}
+
+	// safely strip unecessary prefix, otherise pollutes CRC
+	if ( !strnicmp( pFilename, ".\\", 2 ) )
+	{
+		pFilename += 2;
+	}
+
+	// scan for CRC collision now, not at runtime
+	CRCEntry_t	crcEntry;
+	crcEntry.fileNameCRC = HashStringCaselessConventional( pFilename );
+	crcEntry.filename = pFilename;
+	int idx = m_ZipCRCList.Find( crcEntry );
+	if ( -1 != idx  )
+	{
+		if ( !V_stricmp( pFilename, m_ZipCRCList[idx].filename.String() ) )
+		{
+			// file has already been added, ignore as succesful
+			return true;
+		}
+
+		Msg( "ERROR: CRC Collision: '%s' with '%s'\n", pFilename, m_ZipCRCList[idx].filename.String() );
+		return false;
+	}
+	else
+	{
+		// add unique entry to lists
+		// must track filenames in a non-sort manner
+		m_ZipCRCList.Insert( crcEntry );
+	}
+
+	// default, no preload entry
+	unsigned short preloadDir = INVALID_PRELOAD_ENTRY;
+
+	if ( bDoPreload )
+	{
+		CUtlBuffer preloadBuffer;
+		bool bHasPreload = GetPreloadBuffer( pFilename, fileBuffer, preloadBuffer );
+		int preloadSize = preloadBuffer.TellMaxPut();
+		if ( bHasPreload && preloadSize > 0 ) 
+		{
+			if ( m_ZipPreloadDirectoryEntries.Count() >= 65534 )
+			{
+				Msg( "ERROR: Preload section FULL!, skipping %s\n", pFilename );			
+				return FALSE;
+			}
+		
+			// Initialize the entry header
+			ZIP_PreloadDirectoryEntry entry;
+			memset( &entry, 0, sizeof( entry ) );
+
+			entry.Length = preloadSize;
+			entry.DataOffset = SetFilePointer( m_hPreloadFile, 0, NULL, FILE_CURRENT );
+
+			// Add the directory entry to the preload table
+			preloadDir = m_ZipPreloadDirectoryEntries.AddToTail( entry );
+
+			// Append the preload data to the preload file
+			DWORD numBytesWritten;
+			BOOL bOK = WriteFile( m_hPreloadFile, preloadBuffer.Base(), preloadSize, &numBytesWritten, NULL );
+			if ( !bOK || preloadSize != numBytesWritten )
+			{
+				Msg( "ERROR: writing %d preload bytes of '%s'\n", preloadSize, pFilename );
+				return false;
+			}
+
+			if ( !g_bQuiet )
+			{
+				// Spew it
+				if ( LZMA_IsCompressed( (unsigned char *)preloadBuffer.Base() ) )
+				{
+					unsigned int actualSize = LZMA_GetActualSize( (unsigned char *)preloadBuffer.Base() );
+					Msg( "Preload: '%s': Compressed:%u Actual:%u\n", pFilename, preloadSize, actualSize );
+				}
+				else
+				{
+					Msg( "Preload: '%s': Length:%u\n", pFilename, preloadSize );
+				}
+			}
+		}
+	}
+
+	unsigned int fileSize = fileBuffer.TellMaxPut();
+	if ( fileSize > 0 )
+	{
+		// order in zip is sorted, not sequential
+		m_pZip->AddBufferToZip( pFilename, fileBuffer.Base(), fileSize, false );
+
+		// track the file in the zip directory to it's preload entry
+		// order in preload is sequential as buffers are added
+		preloadRemap_t	remap;
+		remap.filename = pFilename;
+		remap.preloadDirIndex = preloadDir;
+		m_ZipPreloadRemapEntries.AddToTail( remap );
+
+		if ( !g_bQuiet )
+		{
+			Msg( "File: '%s': Length:%u\n", pFilename, fileSize );
+		}
+	}
+
+	return true;
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: Load a file and add it to the zip
+//-----------------------------------------------------------------------------
+bool CXZipTool::AddFile( const char *pFilename, bool bDoPreload )
+{
+	if ( !m_pZip )
+	{
+		return false;
+	}
+
+	FILE* pFile = fopen( pFilename, "rb" );
+	if( !pFile )
+	{
+		Msg( "ERROR: failed to open file: '%s'\n", pFilename );
+		return false;
+	}
+
+	// Get the length of the file
+	fseek( pFile, 0, SEEK_END );
+	unsigned fileSize = ftell( pFile );
+	fseek( pFile, 0, SEEK_SET);
+
+	// read file to buffer
+	CUtlBuffer fileBuffer;
+	fileBuffer.EnsureCapacity( fileSize );
+	fread( fileBuffer.Base(), fileSize, 1, pFile );
+	fclose( pFile );
+	fileBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, fileSize );
+
+	return AddBuffer( pFilename, fileBuffer, bDoPreload );
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: Add the preload section and save out the zip file
+//-----------------------------------------------------------------------------
+bool CXZipTool::End()
+{
+	if ( !m_pZip )
+	{
+		return false;
+	}
+
+	// Add the preload section to the zip
+	if ( m_ZipPreloadDirectoryEntries.Count() )
+	{
+		CUtlBuffer sectionBuffer;
+		CByteswap byteSwap;
+
+		// pc tools write 360 native data
+		byteSwap.ActivateByteSwapping( IsPC() );
+
+		// determine the preload data footprint
+		unsigned int preloadDataSize = SetFilePointer( m_hPreloadFile, 0, NULL, FILE_CURRENT );
+
+		// finalize header
+		m_ZipPreloadHeader.DirectoryEntries = m_ZipPreloadRemapEntries.Count();
+		m_ZipPreloadHeader.PreloadDirectoryEntries = m_ZipPreloadDirectoryEntries.Count();
+
+		// determine the total section size ( treated as a single file inside zip )
+		unsigned int sectionSize = sizeof( ZIP_PreloadHeader ) + 
+						m_ZipPreloadHeader.PreloadDirectoryEntries * sizeof( ZIP_PreloadDirectoryEntry ) +
+						m_ZipPreloadHeader.DirectoryEntries * sizeof( unsigned short ) +
+						preloadDataSize;
+		sectionSize = AlignValue( sectionSize, m_ZipPreloadHeader.Alignment );
+		sectionBuffer.EnsureCapacity( sectionSize );
+
+		// save data in order
+		// save the header
+		byteSwap.SwapFieldsToTargetEndian( &m_ZipPreloadHeader );
+		sectionBuffer.Put( &m_ZipPreloadHeader, sizeof( ZIP_PreloadHeader ) );
+
+		// fixup and save the preload directory
+		for ( int i=0; i<m_ZipPreloadDirectoryEntries.Count(); i++ )
+		{
+			ZIP_PreloadDirectoryEntry entry = m_ZipPreloadDirectoryEntries[i];
+			byteSwap.SwapFieldsToTargetEndian( &entry );
+			sectionBuffer.Put( &entry, sizeof( ZIP_PreloadDirectoryEntry ) );
+		}
+
+		// generate remap table
+		char			fileName[MAX_PATH];
+		int				fileSize;
+		int				zipIndex = -1;
+		unsigned short	*pRemapTable = (unsigned short *)malloc( m_ZipPreloadRemapEntries.Count() * sizeof( unsigned short ) );
+		for ( int i=0; i<m_ZipPreloadRemapEntries.Count(); i++ )
+		{
+			// zip files get iterated in the same order they are serialized to disk
+			fileName[0] = '\0';
+			fileSize = 0;
+			zipIndex = m_pZip->GetNextFilename( zipIndex, fileName, sizeof( fileName ), fileSize );
+
+			// find the file in the preload remap table
+			bool bFound = false;
+			int j;
+			for ( j=0; j<m_ZipPreloadRemapEntries.Count(); j++ )
+			{
+				if ( !Q_stricmp( fileName, m_ZipPreloadRemapEntries[j].filename.String() ) )
+				{
+					bFound = true;
+					break;
+				}
+			}
+			if ( !bFound )
+			{
+				// shouldn't happen, every file in the zip has a matching preload remap entry, that is valid or marked invalid
+				Msg( "ERROR: file '%s' was expected to have an entry in preload table\n", fileName );
+			}
+
+			// the remap table is used to go find a file's preload data (if available)
+			pRemapTable[i] = m_ZipPreloadRemapEntries[j].preloadDirIndex;
+		}	
+		for ( int i=0; i<m_ZipPreloadRemapEntries.Count(); i++ )
+		{
+			unsigned short s = pRemapTable[i];
+			sectionBuffer.PutShort( BigShort( s ) );
+		}
+		free( pRemapTable );		
+
+		// get and save preload data
+		void *pPreloadData = malloc( preloadDataSize );
+		SetFilePointer( m_hPreloadFile, 0, NULL, FILE_BEGIN );
+		DWORD numBytesRead;
+		BOOL bOK = ReadFile( m_hPreloadFile, pPreloadData, preloadDataSize, &numBytesRead, NULL );
+		if ( !bOK || numBytesRead != preloadDataSize )
+		{
+			Msg( "ERROR: failed to read %d bytes from temporary preload file\n", preloadDataSize );		
+		}
+		CloseHandle( m_hPreloadFile );
+		m_hPreloadFile = INVALID_HANDLE_VALUE;
+		
+		sectionBuffer.Put( pPreloadData, preloadDataSize );
+		free( pPreloadData );
+
+		// cannot have written more than was pre-calced, unless code was broken
+		Assert( (unsigned int)sectionBuffer.TellMaxPut() <= sectionSize );
+		while( (unsigned int)sectionBuffer.TellMaxPut() < sectionSize )
+		{
+			// pad to final alignment
+			sectionBuffer.PutChar( 0 );	
+		}
+
+		m_pZip->AddBufferToZip( PRELOAD_SECTION_NAME, sectionBuffer.Base(), sectionBuffer.TellMaxPut(), false );
+	}
+	else
+	{
+		// Clear the preload section placeholder
+		m_pZip->RemoveFileFromZip( PRELOAD_SECTION_NAME );
+	}
+
+	m_pZip->SaveToDisk( m_hOutputZipFile );
+
+	Reset();
+	
+	return true;
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: Create the zip file
+//-----------------------------------------------------------------------------
+bool CXZipTool::Begin( const char *pZipFileName, unsigned int alignment )
+{
+	// get the volume of the target zip
+	char drivePath[MAX_PATH];
+	_splitpath( pZipFileName, drivePath, NULL, NULL, NULL );
+
+	m_pZip = IZip::CreateZip( drivePath, true );
+
+	if ( alignment )
+	{
+		// making an aligned zip that uses an optimized (but incompatible) format
+		m_pZip->ForceAlignment( true, false, alignment );
+	}
+
+	// Open the output file
+	m_hOutputZipFile = CreateFile( pZipFileName, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL );
+	if ( m_hOutputZipFile == INVALID_HANDLE_VALUE )
+	{
+		Msg( "ERROR: failed to create zip file '%s'\n", pZipFileName );
+		return false;
+	}
+	
+	// Create a temporary file for storing the preloaded data
+	scriptlib->MakeTemporaryFilename( g_szModPath, m_PreloadFilename, sizeof( m_PreloadFilename ) );	
+	m_hPreloadFile = CreateFile( m_PreloadFilename, GENERIC_READ|GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL );
+	if ( m_hPreloadFile == INVALID_HANDLE_VALUE )
+	{
+		Msg( "ERROR: failed to create temporary file '%s' for preload data\n", m_PreloadFilename );
+		CloseHandle( m_hOutputZipFile );
+		m_hOutputZipFile = INVALID_HANDLE_VALUE;
+		return false;
+	}
+	memset( &m_ZipPreloadHeader, 0, sizeof( ZIP_PreloadHeader ) );
+
+	m_ZipPreloadHeader.Version = PRELOAD_HDR_VERSION;
+	m_ZipPreloadHeader.Alignment = alignment;
+
+	// Add a placeholder for the preload section
+	m_pZip->AddBufferToZip( PRELOAD_SECTION_NAME, NULL, 0, false );
+	preloadRemap_t remap;
+	remap.filename = PRELOAD_SECTION_NAME;
+	remap.preloadDirIndex = INVALID_PRELOAD_ENTRY;
+	m_ZipPreloadRemapEntries.AddToTail( remap );
+
+	return true;
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: Dump the preload contents
+//-----------------------------------------------------------------------------
+void CXZipTool::SpewPreloadInfo( const char *pZipName )
+{
+	IZip *pZip = IZip::CreateZip( NULL, true );
+	
+	HANDLE hZipFile = pZip->ParseFromDisk( pZipName );
+	if ( !hZipFile )
+	{
+		Msg( "Bad or missing zip file, failed to open '%s'\n", pZipName );
+		return;
+	}
+
+	CUtlBuffer preloadBuffer;
+	if ( !pZip->ReadFileFromZip( hZipFile, PRELOAD_SECTION_NAME, false, preloadBuffer ) )
+	{
+		Msg( "No preload info for '%s'\n", pZipName );
+		return;
+	}
+
+	preloadBuffer.ActivateByteSwapping( IsPC() );
+
+	ZIP_PreloadHeader header;
+	preloadBuffer.GetObjects( &header );
+
+	// get the dir table
+	ZIP_PreloadDirectoryEntry *pDir = (ZIP_PreloadDirectoryEntry *)malloc( header.PreloadDirectoryEntries * sizeof( ZIP_PreloadDirectoryEntry ) );
+	preloadBuffer.GetObjects( pDir, header.PreloadDirectoryEntries );
+
+	// get the remap table
+	unsigned short *pRemap = (unsigned short *)malloc( header.DirectoryEntries * sizeof( unsigned short ) );
+	for ( unsigned int i = 0; i < header.DirectoryEntries; i++ )
+	{
+		pRemap[i] = preloadBuffer.GetShort();
+	}
+
+	int zipIndex = -1;
+	int fileSize;
+	char fileName[MAX_PATH];
+
+	// iterate preload entries sequentially
+	CUtlDict< unsigned int, int > sizes( true );
+	for ( unsigned int i = 0; i < header.DirectoryEntries; i++ )
+	{
+		fileName[0] = '\0';
+		fileSize = 0;
+		zipIndex = pZip->GetNextFilename( zipIndex, fileName, sizeof( fileName ), fileSize );
+
+		unsigned short zipPreloadDirIndex = pRemap[i];
+		if ( zipPreloadDirIndex == INVALID_PRELOAD_ENTRY )
+		{
+			continue;
+		}
+
+		Msg( "Offset: 0x%8.8x Length: %5d %s (%d)\n", pDir[zipPreloadDirIndex].DataOffset, pDir[zipPreloadDirIndex].Length, fileName, fileSize );
+
+		// total preload sizes by extension
+		const char *pExt = V_GetFileExtension( fileName );
+		if ( !pExt )
+		{
+			pExt = "???";
+		}
+		int iIndex = sizes.Find( pExt );
+		if ( iIndex == sizes.InvalidIndex() )
+		{
+			iIndex = sizes.Insert( pExt );
+			sizes[iIndex] = 0;
+		}
+		sizes[iIndex] += pDir[zipPreloadDirIndex].Length;
+	}
+
+	Msg( "\n" );
+	Msg( "Preload Size:    %.2f MB\n", (float)preloadBuffer.TellMaxPut()/(1024.0f * 1024.0f) );
+	Msg( "Zip Entries:     %d\n", header.DirectoryEntries );
+	Msg( "Preload Entries: %d\n", header.PreloadDirectoryEntries );
+
+	// dump each extension's total size, necessary for debugging who is the largest contributor
+	for ( int i = 0; i < sizes.Count(); i++ )
+	{
+		Msg( "Extension: '%3s' %d bytes (%.2f%s)\n", sizes.GetElementName( i ), sizes[i], (float)sizes[i]/(float)preloadBuffer.TellMaxPut() * 100.0f, "%%" );
+	}
+	Msg( "\n" );
+
+	free( pRemap );
+	free( pDir );
+
+	IZip::ReleaseZip( pZip );
+}