From 3bf9df6b2785fa6d951086978a3e66f49427166a Mon Sep 17 00:00:00 2001 From: FluorescentCIAAfricanAmerican <0934gj3049fk@protonmail.com> Date: Wed, 22 Apr 2020 12:56:21 -0400 Subject: 1 --- utils/shadercompile/shadercompile.cpp | 2699 +++++++++++++++++++++++++++++++++ 1 file changed, 2699 insertions(+) create mode 100644 utils/shadercompile/shadercompile.cpp (limited to 'utils/shadercompile/shadercompile.cpp') diff --git a/utils/shadercompile/shadercompile.cpp b/utils/shadercompile/shadercompile.cpp new file mode 100644 index 0000000..60eb68a --- /dev/null +++ b/utils/shadercompile/shadercompile.cpp @@ -0,0 +1,2699 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +//=============================================================================// +// vmpi_bareshell.cpp : Defines the entry point for the console application. +// + +#include +#include +#include +#include "vmpi.h" +#include "filesystem.h" +#include "vmpi_filesystem.h" +#include "vmpi_distribute_work.h" +#include "vmpi_tools_shared.h" +#include "cmdlib.h" +#include "utlvector.h" +#include "Utlhash.h" +#include "UtlBuffer.h" +#include "utlstring.h" +#include "utlbinaryblock.h" +#include "tier2/utlstreambuffer.h" +#include "UtlLinkedList.h" +#include "UtlStringMap.h" +#include "tier0/icommandline.h" +#include "tier1/strtools.h" +#include "vstdlib/jobthread.h" +#include "threads.h" +#include "tier0/dbg.h" +#include "tier1/smartptr.h" +#include "interface.h" +#include "ishadercompiledll.h" +#include +#include "io.h" +#include +#include +#include "materialsystem/shader_vcs_version.h" +#include "ilaunchabledll.h" +#include +#include "utlnodehash.h" +#include "lzma/lzma.h" +#include "mathlib/mathlib.h" +#include "tier1/checksum_crc.h" +#include "tier0/tslist.h" +#include "tools_minidump.h" + +#include "cmdsink.h" +#include "d3dxfxc.h" +#include "subprocess.h" +#include "cfgprocessor.h" + + +// Type conversions should be controlled by programmer explicitly - shadercompile makes use of 64-bit integer arithmetics +#pragma warning( error : 4244 ) + +static inline uint32 uint64_as_uint32( uint64 x ) +{ + Assert( x < uint64( uint32( ~0 ) ) ); + return uint32( x ); +} + +static inline UtlSymId_t int_as_symid( int x ) +{ + Assert( ( sizeof( UtlSymId_t ) >= sizeof( int ) ) || ( x >= 0 && x < ( int )( unsigned int )( UtlSymId_t(~0) ) ) ); + return UtlSymId_t( x ); +} + + +// VMPI packets +#define STARTWORK_PACKETID 5 +#define WORKUNIT_PACKETID 6 +#define ERRMSG_PACKETID 7 +#define SHADERHADERROR_PACKETID 8 +#define MACHINE_NAME 9 + +#ifdef _DEBUG +//#define DEBUGFP +#endif + + +// Dealing with job list +namespace +{ + +CArrayAutoPtr< CfgProcessor::CfgEntryInfo > g_arrCompileEntries; +uint64 g_numShaders = 0, g_numCompileCommands = 0, g_numStaticCombos = 0; +uint64 g_nStaticCombosPerWorkUnit = 0, g_numCompletedStaticCombos = 0, g_numCommandsCompleted = 0; +uint64 g_numSkippedStaticCombos = 0; + +CfgProcessor::CfgEntryInfo const * GetEntryByStaticComboNum( uint64 nStaticCombo, uint64 *pnStaticCombo ) +{ + CfgProcessor::CfgEntryInfo const *pInfo; + uint64 nRemainStaticCombos = nStaticCombo; + + for ( pInfo = g_arrCompileEntries.Get(); pInfo && pInfo->m_szName; ++ pInfo ) + { + if ( nRemainStaticCombos >= pInfo->m_numStaticCombos ) + nRemainStaticCombos -= pInfo->m_numStaticCombos; + else + break; + } + + if ( pnStaticCombo ) + *pnStaticCombo = nRemainStaticCombos; + + return pInfo; +} + +}; // `anonymous` namespace + +char * PrettyPrintNumber( uint64 k ) +{ + static char chCompileString[50] = {0}; + char *pchPrint = chCompileString + sizeof( chCompileString ) - 3; + for ( uint64 j = 0; k > 0; k /= 10, ++ j ) + { + ( j && !( j % 3 ) ) ? ( * pchPrint -- = ',' ) : 0; + * pchPrint -- = '0' + char( k % 10 ); + } + ( * ++ pchPrint ) ? 0 : ( * pchPrint = 0 ); + return pchPrint; +} + + +const char *g_pShaderPath = NULL; +char g_WorkerTempPath[MAX_PATH]; +char g_ExeDir[MAX_PATH]; +#ifdef DEBUGFP +FILE *g_WorkerDebugFp = NULL; +#endif +bool g_bGotStartWorkPacket = false; +double g_flStartTime; +bool g_bVerbose = false; +bool g_bIsX360 = false; +bool g_bSuppressWarnings = false; + +FORCEINLINE long AsTargetLong( long x ) { return ( ( g_bIsX360 ) ? ( BigLong( x ) ) : ( x ) ); } + + +struct ShaderInfo_t +{ + ShaderInfo_t() { memset( this, 0, sizeof( *this ) ); } + + uint64 m_nShaderCombo; + uint64 m_nTotalShaderCombos; + const char *m_pShaderName; + const char *m_pShaderSrc; + unsigned m_CentroidMask; + uint64 m_nDynamicCombos; + uint64 m_nStaticCombo; + unsigned m_Flags; // from IShader.h + char m_szShaderModel[ 12 ]; +}; + +void Shader_ParseShaderInfoFromCompileCommands( CfgProcessor::CfgEntryInfo const *pEntry, ShaderInfo_t &shaderInfo ); + +struct CByteCodeBlock +{ + + CByteCodeBlock *m_pNext, *m_pPrev; + int m_nCRC32; + uint64 m_nComboID; + size_t m_nCodeSize; + uint8 *m_ByteCode; + + CByteCodeBlock( void ) + { + m_ByteCode = NULL; + } + + CByteCodeBlock( void const *pByteCode, size_t nCodeSize, uint64 nComboID ) + { + m_ByteCode = new uint8[nCodeSize]; + m_nComboID = nComboID; + m_nCodeSize = nCodeSize; + memcpy( m_ByteCode, pByteCode, nCodeSize ); + m_nCRC32 = CRC32_ProcessSingleBuffer( pByteCode, nCodeSize ); + } + + ~CByteCodeBlock( void ) + { + if ( m_ByteCode ) + delete[] m_ByteCode; + } + +}; + +static int __cdecl CompareDynamicComboIDs( CByteCodeBlock * const *pA, CByteCodeBlock * const *pB ) +{ + if ( (*pA)->m_nComboID < (*pB)->m_nComboID ) + return -1; + if ( (*pA)->m_nComboID > (*pB)->m_nComboID ) + return 1; + return 0; +} + + +struct CStaticCombo // all the data for one static combo +{ + CStaticCombo *m_pNext, *m_pPrev; + + uint64 m_nStaticComboID; + + CUtlVector< CByteCodeBlock* > m_DynamicCombos; + + struct PackedCode : protected CArrayAutoPtr { + size_t GetLength() const { if( uint8 *pb = Get() ) return *reinterpret_cast( pb ); else return 0; } + uint8 *GetData() const { if( uint8 *pb = Get() ) return pb + sizeof( size_t ); else return NULL; } + uint8 *AllocData( size_t len ) { Delete(); if ( len ) { Attach( new uint8[ len + sizeof( size_t ) ] ); *reinterpret_cast( Get() ) = len; } return GetData(); } + } m_abPackedCode; // Packed code for entire static combo + + uint64 Key( void ) const + { + return m_nStaticComboID; + } + + CStaticCombo( uint64 nComboID ) + { + m_nStaticComboID = nComboID; + } + + ~CStaticCombo( void ) + { + m_DynamicCombos.PurgeAndDeleteElements(); + } + + void AddDynamicCombo( uint64 nComboID, void const *pComboData, size_t nCodeSize ) + { + CByteCodeBlock *pNewBlock = new CByteCodeBlock( pComboData, nCodeSize, nComboID ); + m_DynamicCombos.AddToTail( pNewBlock ); + } + + void SortDynamicCombos( void ) + { + m_DynamicCombos.Sort( CompareDynamicComboIDs ); + } + + uint8 *AllocPackedCodeBlock( size_t nPackedCodeSize ) + { + return m_abPackedCode.AllocData( nPackedCodeSize ); + } + +}; + +typedef CUtlNodeHash StaticComboNodeHash_t; + +template <> +inline StaticComboNodeHash_t **Construct( StaticComboNodeHash_t ** pMemory ) +{ + return ::new( pMemory ) StaticComboNodeHash_t *( NULL ); // Explicitly new with NULL +} + +struct CShaderMap : public CUtlStringMap { + ; +} g_ShaderByteCode; + + + +CStaticCombo * StaticComboFromDictAdd( char const *pszShaderName, uint64 nStaticComboId ) +{ + StaticComboNodeHash_t *& rpNodeHash = g_ShaderByteCode[ pszShaderName ]; + if ( !rpNodeHash ) + { + rpNodeHash = new StaticComboNodeHash_t; + } + + // search for this static combo. make it if not found + CStaticCombo *pStaticCombo = rpNodeHash->FindByKey( nStaticComboId ); + if ( !pStaticCombo ) + { + pStaticCombo = new CStaticCombo( nStaticComboId ); + rpNodeHash->Add( pStaticCombo ); + } + + return pStaticCombo; +} + +CStaticCombo * StaticComboFromDict( char const *pszShaderName, uint64 nStaticComboId ) +{ + if ( StaticComboNodeHash_t *pNodeHash = g_ShaderByteCode[ pszShaderName ] ) + return pNodeHash->FindByKey( nStaticComboId ); + else + return NULL; +} + + + +CUtlStringMap g_ShaderToShaderInfo; + +class CompilerMsgInfo +{ +public: + CompilerMsgInfo() : m_numTimesReported( 0 ) {} + +public: + void SetMsgReportedCommand( char const *szCommand, int numTimesReported = 1 ) { if ( !m_numTimesReported ) m_sFirstCommand = szCommand; m_numTimesReported += numTimesReported; } + +public: + char const * GetFirstCommand() const { return m_sFirstCommand.String(); } + int GetNumTimesReported() const { return m_numTimesReported; } + +protected: + CUtlString m_sFirstCommand; + int m_numTimesReported; +}; + +CUtlStringMap g_Master_ShaderHadError; +CUtlStringMap g_Master_ShaderWrittenToDisk; +CUtlStringMap g_Master_CompilerMsgInfo; + +namespace Threading +{ + +enum Mode { eSingleThreaded = 0, eMultiThreaded = 1 }; + +// A special object that makes single-threaded code incur no penalties +// and multithreaded code to be synchronized properly. +template < class MT_MUTEX_TYPE = CThreadFastMutex > +class CSwitchableMutex +{ +public: + +public: + FORCEINLINE explicit CSwitchableMutex( Mode eMode, MT_MUTEX_TYPE *pMtMutex = NULL ) : m_pMtx( pMtMutex ), m_pUseMtx( eMode ? pMtMutex : NULL ) {} + +public: + FORCEINLINE void SetMtMutex( MT_MUTEX_TYPE *pMtMutex ) { m_pMtx = pMtMutex; m_pUseMtx = ( m_pUseMtx ? pMtMutex : NULL ); } + FORCEINLINE void SetThreadedMode( Mode eMode ) { m_pUseMtx = ( eMode ? m_pMtx : NULL ); } + +public: + FORCEINLINE void Lock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->Lock(); } + FORCEINLINE void Unlock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->Unlock(); } + + FORCEINLINE bool TryLock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->TryLock(); else return true; } + FORCEINLINE bool AssertOwnedByCurrentThread() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->AssertOwnedByCurrentThread(); else return true; } + FORCEINLINE void SetTrace( bool b ) { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->SetTrace( b ); } + + FORCEINLINE uint32 GetOwnerId() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->GetOwnerId(); else return 0; } + FORCEINLINE int GetDepth() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->GetDepth(); else return 0; } + +private: + MT_MUTEX_TYPE *m_pMtx; + CInterlockedPtr< MT_MUTEX_TYPE > m_pUseMtx; +}; + + +namespace Private +{ + + typedef CThreadMutex MtMutexType_t; + MtMutexType_t g_mtxSyncObjMT; + +}; // namespace Private + + +CSwitchableMutex< Private::MtMutexType_t > g_mtxGlobal( eSingleThreaded, &Private::g_mtxSyncObjMT ); + + +class CGlobalMutexAutoLock +{ +public: + CGlobalMutexAutoLock() { g_mtxGlobal.Lock(); } + ~CGlobalMutexAutoLock() { g_mtxGlobal.Unlock(); } +}; + +}; // namespace Threading + +// Access to global data should be synchronized by these global locks +#define GLOBAL_DATA_MTX_LOCK() Threading::g_mtxGlobal.Lock() +#define GLOBAL_DATA_MTX_UNLOCK() Threading::g_mtxGlobal.Unlock() +#define GLOBAL_DATA_MTX_LOCK_AUTO Threading::CGlobalMutexAutoLock UNIQUE_ID; + + + +CDispatchReg g_DistributeWorkReg( WORKUNIT_PACKETID, DistributeWorkDispatch ); + +unsigned long VMPI_Stats_GetJobWorkerID( void ) +{ + return 0; +} + + +bool StartWorkDispatch( MessageBuffer *pBuf, int iSource, int iPacketID ) +{ + g_bGotStartWorkPacket = true; + return true; +} + +CDispatchReg g_StartWorkReg( STARTWORK_PACKETID, StartWorkDispatch ); + +// Consume all characters for which (isspace) is true +template < typename T > +char * ConsumeCharacters( char *szString, T pred ) +{ + if ( szString ) + { + while ( *szString && pred( *szString ) ) + { + ++ szString; + } + } + + return szString; +} + +char * FindNext( char *szString, char *szSearchSet ) +{ + bool bFound = (szString == NULL); + char *szNext = NULL; + + if ( szString && szSearchSet ) + { + for ( ; *szSearchSet; ++ szSearchSet ) + { + if ( char *szTmp = strchr( szString, *szSearchSet ) ) + { + szNext = bFound ? ( min( szNext, szTmp ) ) : szTmp; + bFound = true; + } + } + } + + return bFound ? szNext : ( szString + strlen( szString ) ); +} + +char * FindLast( char *szString, char *szSearchSet ) +{ + bool bFound = (szString != NULL); + char *szNext = NULL; + + if ( szString && szSearchSet ) + { + for ( ; *szSearchSet; ++ szSearchSet ) + { + if ( char *szTmp = strrchr( szString, *szSearchSet ) ) + { + szNext = bFound ? ( max( szNext, szTmp ) ) : szTmp; + bFound = true; + } + } + } + + return bFound ? szNext : ( szString + strlen( szString ) ); +} + +void ErrMsgDispatchMsgLine( char const *szCommand, char *szMsgLine, char const *szShaderName = NULL ) +{ + // When the filename is specified in front of the message, make sure it is truncated to the bare name only + if ( V_isalpha( *szMsgLine ) && szMsgLine[1] == ':' ) + { + // Preceded by drive letter + szMsgLine += 2; + } + + // Trim the path from the msg + // e.g. make string + // c:\temp\shadercompiletemp\1234\myfile.fxc(435): warning X3083: Truncating ... + // look like + // myfile.fxc(435): warning X3083: Truncating ... + // which will be both readable and same coming from different worker machines + char *szEndFileLinePlant = FindNext( szMsgLine, ":" ); + if ( ':' == *szEndFileLinePlant ) + { + *szEndFileLinePlant = 0; + if ( char *szLastSlash = FindLast( szMsgLine, "\\/" ) ) + { + if ( *szLastSlash ) + { + *szLastSlash = 0; + szMsgLine = szLastSlash + 1; + } + } + *szEndFileLinePlant = ':'; + } + + // If the shader file name is not given in the message add it + if ( szShaderName ) + { + static char chFitLongMsgLine[4096]; + + if ( *szMsgLine == '(' ) + { + sprintf( chFitLongMsgLine, "%s%s", szShaderName, szMsgLine ); + szMsgLine = chFitLongMsgLine; + } + else if ( !strncmp( szMsgLine, "memory(", 7 ) ) + { + sprintf( chFitLongMsgLine, "%s%s", szShaderName, szMsgLine+6 ); + szMsgLine = chFitLongMsgLine; + } + } + + // Now store the message with the command it was generated from + g_Master_CompilerMsgInfo[ szMsgLine ].SetMsgReportedCommand( szCommand ); +} + +void ErrMsgDispatchInt( char *szMessage, char const *szShaderName = NULL ) +{ + // First line is the command number "szCommand" + char *szCommand = ConsumeCharacters( szMessage, isspace ); + char *szMessageListing = FindNext(szCommand, "\r\n"); + char chTerminator = *szMessageListing; + *( szMessageListing ++ ) = 0; + + // Now come the command lines actually + while ( chTerminator ) + { + char *szMsgText = ConsumeCharacters( szMessageListing, isspace ); + szMessageListing = FindNext( szMsgText, "\r\n" ); + chTerminator = *szMessageListing; + *( szMessageListing ++ ) = 0; + + if( *szMsgText ) + { + // Trim command at redirection character if present + * FindNext( szCommand, ">" ) = 0; + ErrMsgDispatchMsgLine( szCommand, szMsgText, szShaderName ); + } + } +} + +// +// BUFFER: +// 1 byte = * = buffer type +// +// string = message +// 1 byte = \n = newline delimiting the message +// +// string = command that first encountered the message +// 1 byte = \n = newline delimiting the command +// +// string = printed number of times the message was encountered +// 1 byte = \n = newline delimiting the number +// +// 1 byte = 0 = null-terminator for the buffer +// +bool ErrMsgDispatch( MessageBuffer *pBuf, int iSource, int iPacketID ) +{ + GLOBAL_DATA_MTX_LOCK_AUTO; + + bool bInvalidPkgRetCode = true; + + // Parse the err msg packet + char *szMsgLine = pBuf->data + 1; + + char *szCommand = FindNext( szMsgLine, "\n" ); + if ( !*szCommand ) + return bInvalidPkgRetCode; + *( szCommand ++ ) = 0; + + char *szNumTimesReported = FindNext( szCommand, "\n" ); + if ( !*szNumTimesReported ) + return bInvalidPkgRetCode; + *( szNumTimesReported ++ ) = 0; + + char *szTerminator = FindNext( szNumTimesReported, "\n" ); + if ( !*szTerminator ) + return bInvalidPkgRetCode; + *( szTerminator ++ ) = 0; + + // Set the msg info + g_Master_CompilerMsgInfo[ szMsgLine ].SetMsgReportedCommand( szCommand, atoi( szNumTimesReported ) ); + + return true; +} + +CDispatchReg g_ErrMsgReg( ERRMSG_PACKETID, ErrMsgDispatch ); + +void ShaderHadErrorDispatchInt( char const *szShader ) +{ + g_Master_ShaderHadError[ szShader ] = true; +} + +// +// BUFFER: +// 1 byte = * = buffer type +// +// string = shader name +// 1 byte = 0 = null-terminator for the name +// +bool ShaderHadErrorDispatch( MessageBuffer *pBuf, int iSource, int iPacketID ) +{ + GLOBAL_DATA_MTX_LOCK_AUTO; + + ShaderHadErrorDispatchInt( pBuf->data + 1 ); + return true; +} + +CDispatchReg g_ShaderHadErrorReg( SHADERHADERROR_PACKETID, ShaderHadErrorDispatch ); + +void DebugOut( const char *pMsg, ... ) +{ + if (g_bVerbose) + { + char msg[2048]; + va_list marker; + va_start( marker, pMsg ); + _vsnprintf( msg, sizeof( msg ), pMsg, marker ); + va_end( marker ); + + Msg( "%s", msg ); + +#ifdef DEBUGFP + fprintf( g_WorkerDebugFp, "%s", msg ); + fflush( g_WorkerDebugFp ); +#endif + } +} + +void Vmpi_Worker_DefaultDisconnectHandler( int procID, const char *pReason ) +{ + Msg( "Master disconnected.\n "); + DebugOut( "Master disconnected.\n" ); + TerminateProcess( GetCurrentProcess(), 1 ); +} + +typedef void ( * DisconnectHandlerFn_t )( int procID, const char *pReason ); +DisconnectHandlerFn_t g_fnDisconnectHandler = Vmpi_Worker_DefaultDisconnectHandler; + +// Worker should implement this so it will quit nicely when the master disconnects. +void MyDisconnectHandler( int procID, const char *pReason ) +{ + // If we're a worker, then it's a fatal error if we lose the connection to the master. + if ( !g_bMPIMaster && g_fnDisconnectHandler ) + { + (* g_fnDisconnectHandler)( procID, pReason ); + } +} + + + +// new format: +// ver# +// total shader combos +// total dynamic combos +// flags +// centroid mask +// total non-skipped static combos +// [ (sorted by static combo id) +// static combo id +// file offset of packed dynamic combo +// ] +// 0xffffffff (sentinel key) +// end of file offset (so can tell compressed size of last combo) +// +// # of duplicate static combos (if version >= 6 ) +// [ (sorted by static combo id) +// static combo id +// id of static bombo which is identical +// ] +// +// each packed dynamic combo for a given static combo is stored as a series of compressed blocks. +// block 1: +// ulong blocksize (high bit set means uncompressed) +// block data +// block2.. +// 0xffffffff indicates no more blocks for this combo +// +// each block, when uncompressed, holds one or more dynamic combos: +// dynamic combo id (full id if v<6, dynamic combo id only id >=6) +// size of shader +// .. +// there is no terminator - the size of the uncompressed shader tells you when to stop + + + + +// this record is then bzip2'd. + +// qsort driver function +// returns negative number if idA is less than idB, positive when idA is greater than idB +// and zero if the ids are equal + +static int __cdecl CompareDupComboIndices( const StaticComboAliasRecord_t *pA, const StaticComboAliasRecord_t *pB ) +{ + if ( pA->m_nStaticComboID < pB->m_nStaticComboID ) + return -1; + if ( pA->m_nStaticComboID > pB->m_nStaticComboID ) + return 1; + return 0; +} + +static void FlushCombos( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer, MessageBuffer *pBuf ) +{ + if ( !pDynamicComboBuffer->TellPut() ) + // Nothing to do here + return; + + size_t nCompressedSize; + uint8 *pCompressedShader = LZMA_OpportunisticCompress( reinterpret_cast ( pDynamicComboBuffer->Base() ), + pDynamicComboBuffer->TellPut(), + &nCompressedSize ); + // high 2 bits of length = + // 00 = bzip2 compressed + // 10 = uncompressed + // 01 = lzma compressed + // 11 = unused + + if ( ! pCompressedShader ) + { + // it grew + long lFlagSize = AsTargetLong( 0x80000000 | pDynamicComboBuffer->TellPut() ); + pBuf->write( &lFlagSize, sizeof( lFlagSize ) ); + pBuf->write( pDynamicComboBuffer->Base(), pDynamicComboBuffer->TellPut() ); + *pnTotalFlushedSize += sizeof( lFlagSize ) + pDynamicComboBuffer->TellPut(); + } + else + { + long lFlagSize = AsTargetLong( 0x40000000 | nCompressedSize ); + pBuf->write( &lFlagSize, sizeof( lFlagSize ) ); + pBuf->write( pCompressedShader, nCompressedSize ); + delete[] pCompressedShader; + *pnTotalFlushedSize += sizeof( lFlagSize ) + nCompressedSize; + } + pDynamicComboBuffer->Clear(); // start over +} + +static void OutputDynamicCombo( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer, + MessageBuffer *pBuf, uint64 nComboID, int nComboSize, + uint8 *pComboCode ) +{ + if ( pDynamicComboBuffer->TellPut() + nComboSize+16 >= MAX_SHADER_UNPACKED_BLOCK_SIZE ) + { + FlushCombos( pnTotalFlushedSize, pDynamicComboBuffer, pBuf ); + } + + pDynamicComboBuffer->PutInt( uint64_as_uint32( nComboID ) ); + pDynamicComboBuffer->PutInt( nComboSize ); +// pDynamicComboBuffer->PutInt( CRC32_ProcessSingleBuffer( pComboCode, nComboSize ) ); + pDynamicComboBuffer->Put( pComboCode, nComboSize ); +} + +static void OutputDynamicComboDup( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer, + MessageBuffer *pBuf, uint64 nComboID, uint64 nBaseCombo ) +{ + if ( pDynamicComboBuffer->TellPut() + 8 >= MAX_SHADER_UNPACKED_BLOCK_SIZE ) + { + FlushCombos( pnTotalFlushedSize, pDynamicComboBuffer, pBuf ); + } + pDynamicComboBuffer->PutInt( uint64_as_uint32( nComboID ) | 0x80000000 ); + pDynamicComboBuffer->PutInt( uint64_as_uint32( nBaseCombo ) ); +} + +void GetVCSFilenames( char *pszMainOutFileName, ShaderInfo_t const &si ) +{ + sprintf( pszMainOutFileName, "%s\\shaders\\fxc", g_pShaderPath ); + + struct _stat buf; + if( _stat( pszMainOutFileName, &buf ) == -1 ) + { + printf( "mkdir %s\n", pszMainOutFileName ); + // doh. . need to make the directory that the vcs file is going to go into. + _mkdir( pszMainOutFileName ); + } + + strcat( pszMainOutFileName, "\\" ); + strcat( pszMainOutFileName, si.m_pShaderName ); + + if ( g_bIsX360 ) + { + strcat( pszMainOutFileName, ".360" ); + } + + strcat( pszMainOutFileName, ".vcs" ); // Different extensions for main output file + + // Check status of vcs file... + if( _stat( pszMainOutFileName, &buf ) != -1 ) + { + // The file exists, let's see if it's writable. + if( !( buf.st_mode & _S_IWRITE ) ) + { + // It isn't writable. . we'd better change its permissions (or check it out possibly) + printf( "Warning: making %s writable!\n", pszMainOutFileName ); + _chmod( pszMainOutFileName, _S_IREAD | _S_IWRITE ); + } + } +} + + +// WriteShaderFiles +// +// should be called either on the main thread or +// on the async writing thread. +// +// So the function WriteShaderFiles should not be reentrant, however the +// data that it uses might be updated by the main thread when built pieces +// are received from the workers. +// +#define STATIC_COMBO_HASH_SIZE 73 + +struct StaticComboAuxInfo_t : StaticComboRecord_t +{ + uint32 m_nCRC32; // CRC32 of packed data + struct CStaticCombo *m_pByteCode; +}; + +static int __cdecl CompareComboIds( const StaticComboAuxInfo_t *pA, const StaticComboAuxInfo_t *pB ) +{ + if ( pA->m_nStaticComboID < pB->m_nStaticComboID ) + return -1; + if ( pA->m_nStaticComboID > pB->m_nStaticComboID ) + return 1; + return 0; +} + +static void WriteShaderFiles( const char *pShaderName ) +{ + if ( !g_Master_ShaderWrittenToDisk.Defined( pShaderName ) ) + g_Master_ShaderWrittenToDisk[ pShaderName ] = true; + else + return; + + bool bShaderFailed = g_Master_ShaderHadError.Defined( pShaderName ); + char const *szShaderFileOperation = bShaderFailed ? "Removing failed" : "Writing"; + + // + // Progress indication + // + if ( g_numCommandsCompleted < g_numCompileCommands ) + { + static char chProgress[] = { '/', '-', '\\', '|' }; + static int iProgressSymbol = 0; + Msg( "\b%c", chProgress[ ( ++ iProgressSymbol ) % 4 ] ); + } + else + { + char chShaderName[33]; + Q_snprintf( chShaderName, 29, "%s...", pShaderName ); + sprintf( chShaderName + sizeof( chShaderName ) - 5, "..." ); + Msg( "\r%s %s \r", szShaderFileOperation, chShaderName ); + } + + // + // Retrieve the data we are going to operate on + // from global variables under lock. + // + GLOBAL_DATA_MTX_LOCK(); + StaticComboNodeHash_t *pByteCodeArray; + { + StaticComboNodeHash_t *&rp = g_ShaderByteCode[pShaderName]; // Get a static combo pointer, reset it as well + pByteCodeArray = rp; + rp = NULL; + + /* + Assert( pByteCodeArray ); + if ( !pByteCodeArray ) + ShaderHadErrorDispatchInt( pShaderName ); + */ + } + ShaderInfo_t shaderInfo = g_ShaderToShaderInfo[pShaderName]; + if ( !shaderInfo.m_pShaderName ) + { + for ( CfgProcessor::CfgEntryInfo const *pAnalyze = g_arrCompileEntries.Get() ; + pAnalyze->m_szName ; + ++ pAnalyze ) + { + if ( !strcmp( pAnalyze->m_szName, pShaderName ) ) + { + Shader_ParseShaderInfoFromCompileCommands( pAnalyze, shaderInfo ); + g_ShaderToShaderInfo[ pShaderName ] = shaderInfo; + break; + } + } + } + GLOBAL_DATA_MTX_UNLOCK(); + + if ( !shaderInfo.m_pShaderName ) + return; + + // + // Shader vcs file name + // + char szVCSfilename[MAX_PATH]; + GetVCSFilenames( szVCSfilename, shaderInfo ); + + if ( bShaderFailed ) + { + DebugOut( "Removing failed shader file \"%s\".\n", szVCSfilename ); + unlink( szVCSfilename ); + return; + } + + if ( !pByteCodeArray ) + return; + + DebugOut( "%s : %I64u combos centroid mask: 0x%x numDynamicCombos: %I64u flags: 0x%x\n", + pShaderName, shaderInfo.m_nTotalShaderCombos, + shaderInfo.m_CentroidMask, shaderInfo.m_nDynamicCombos, shaderInfo.m_Flags ); + + // + // Static combo headers + // + CUtlVector< StaticComboAuxInfo_t > StaticComboHeaders; + + StaticComboHeaders.EnsureCapacity( 1 + pByteCodeArray->Count() ); // we know how much ram we need + + CUtlVector< int > comboIndicesHashedByCRC32[STATIC_COMBO_HASH_SIZE]; + CUtlVector< StaticComboAliasRecord_t > duplicateCombos; + + // now, lets fill in our combo headers, sort, and write + for( int nChain = 0 ; nChain < NELEMS( pByteCodeArray->m_HashChains) ; nChain++ ) + { + for( CStaticCombo *pStatic = pByteCodeArray->m_HashChains[ nChain ].m_pHead; + pStatic; + pStatic = pStatic->m_pNext ) + { + if ( pStatic->m_abPackedCode.GetLength() ) + { + StaticComboAuxInfo_t Hdr; + Hdr.m_nStaticComboID = uint64_as_uint32( pStatic->m_nStaticComboID ); + Hdr.m_nFileOffset = 0; // fill in later + Hdr.m_nCRC32 = CRC32_ProcessSingleBuffer( pStatic->m_abPackedCode.GetData(), pStatic->m_abPackedCode.GetLength() ); + int nHashIdx = Hdr.m_nCRC32 % STATIC_COMBO_HASH_SIZE; + Hdr.m_pByteCode = pStatic; + // now, see if we have an identical static combo + bool bIsDuplicate = false; + for( int i = 0; i < comboIndicesHashedByCRC32[nHashIdx].Count() ; i++ ) + { + StaticComboAuxInfo_t const &check = StaticComboHeaders[comboIndicesHashedByCRC32[nHashIdx][i]]; + if ( + ( check.m_nCRC32 == Hdr.m_nCRC32 ) && + ( check.m_pByteCode->m_abPackedCode.GetLength() == pStatic->m_abPackedCode.GetLength() ) && + ( memcmp( check.m_pByteCode->m_abPackedCode.GetData(), pStatic->m_abPackedCode.GetData(), check.m_pByteCode->m_abPackedCode.GetLength() ) == 0 ) + ) + { + // this static combo is the same as another one!! + StaticComboAliasRecord_t aliasHdr; + aliasHdr.m_nStaticComboID = Hdr.m_nStaticComboID; + aliasHdr.m_nSourceStaticCombo = check.m_nStaticComboID; + duplicateCombos.AddToTail( aliasHdr ); + bIsDuplicate = true; + break; + } + } + + if ( ! bIsDuplicate ) + { + StaticComboHeaders.AddToTail( Hdr ); + comboIndicesHashedByCRC32[nHashIdx].AddToTail( StaticComboHeaders.Count() - 1 ); + } + } + } + } + // add sentinel key + StaticComboAuxInfo_t Hdr; + Hdr.m_nStaticComboID = 0xffffffff; + Hdr.m_nFileOffset = 0; + StaticComboHeaders.AddToTail( Hdr ); + + // now, sort. sentinel key will end up at end + StaticComboHeaders.Sort( CompareComboIds ); + + // Set the CRC to zero for now. . will patch in copyshaders.pl with the correct CRC. + unsigned int crc32 = 0; + + // + // Shader file stream buffer + // + CUtlStreamBuffer ShaderFile( szVCSfilename, NULL ); // Streaming buffer for vcs file (since this can blow memory) + ShaderFile.SetBigEndian( g_bIsX360 ); // Swap the header bytes to X360 format + + // ------ Header -------------- + ShaderFile.PutInt( SHADER_VCS_VERSION_NUMBER ); // Version + ShaderFile.PutInt( uint64_as_uint32( shaderInfo.m_nTotalShaderCombos ) ); + ShaderFile.PutInt( uint64_as_uint32( shaderInfo.m_nDynamicCombos ) ); + ShaderFile.PutUnsignedInt( shaderInfo.m_Flags ); + ShaderFile.PutUnsignedInt( shaderInfo.m_CentroidMask ); + ShaderFile.PutUnsignedInt( StaticComboHeaders.Count() ); + ShaderFile.PutUnsignedInt( crc32 ); + + // static combo dictionary + int nDictionaryOffset= ShaderFile.TellPut(); + + // we will re write this one we know the offsets + ShaderFile.Put( StaticComboHeaders.Base(), sizeof( StaticComboRecord_t ) * StaticComboHeaders.Count() ); // dummy write, 8 bytes per static combo + + ShaderFile.PutUnsignedInt( duplicateCombos.Count() ); + // now, write out all duplicate header records + // sort duplicate combo records for binary search + duplicateCombos.Sort( CompareDupComboIndices ); + + for( int i = 0; i < duplicateCombos.Count(); i++ ) + { + ShaderFile.PutUnsignedInt( duplicateCombos[i].m_nStaticComboID ); + ShaderFile.PutUnsignedInt( duplicateCombos[i].m_nSourceStaticCombo ); + } + + // now, write out all static combos + for( int i=0 ; iFindByKey( SRec.m_nStaticComboID ); + Assert( pStatic ); + + // Put the packed chunk of code for this static combo + if ( size_t nPackedLen = pStatic->m_abPackedCode.GetLength() ) + ShaderFile.Put( pStatic->m_abPackedCode.GetData(), nPackedLen ); + + ShaderFile.PutInt( 0xffffffff ); // end of dynamic combos + } + + if ( g_bIsX360 ) + { + SRec.m_nFileOffset = BigLong( SRec.m_nFileOffset ); + SRec.m_nStaticComboID = BigLong( SRec.m_nStaticComboID ); + } + } + ShaderFile.Close(); + + // + // Re-writing the combo header + // + { + FILE *Handle=fopen( szVCSfilename, "rb+" ); + if (! Handle ) + printf(" failed to re-open %s\n",szVCSfilename ); + + fseek( Handle, nDictionaryOffset, SEEK_SET ); + + // now, rewrite header. data is already byte-swapped appropriately + for( int i = 0; i < StaticComboHeaders.Count(); i++ ) + { + fwrite( &( StaticComboHeaders[i].m_nStaticComboID ), 4, 1, Handle ); + fwrite( &( StaticComboHeaders[i].m_nFileOffset ), 4, 1, Handle ); + } + fclose( Handle ); + } + + // Finalize, free memory + delete pByteCodeArray; + + if ( g_numCommandsCompleted >= g_numCompileCommands ) + { + Msg( "\r \r" ); + } +} + +// pBuf is ready to read the results written to the buffer in ProcessWorkUnitFn. +// work is done. .master gets it back this way. +// compiled code in pBuf +void Master_ReceiveWorkUnitFn( uint64 iWorkUnit, MessageBuffer *pBuf, int iWorker ) +{ + GLOBAL_DATA_MTX_LOCK_AUTO; + + uint64 comboStart = iWorkUnit * g_nStaticCombosPerWorkUnit; + uint64 comboEnd = comboStart + g_nStaticCombosPerWorkUnit; + comboEnd = min( g_numStaticCombos, comboEnd ); + + char const *chLastShaderName = ""; + ShaderInfo_t siLastShaderInfo; + memset( &siLastShaderInfo, 0, sizeof( siLastShaderInfo ) ); + siLastShaderInfo.m_pShaderName = chLastShaderName; + + uint64 nComboOfTheEntry = 0; + CfgProcessor::CfgEntryInfo const *pEntry = GetEntryByStaticComboNum( comboStart, &nComboOfTheEntry ); + nComboOfTheEntry = pEntry->m_numStaticCombos - 1 - nComboOfTheEntry; + + for( uint64 iCombo = comboStart; iCombo ++ < comboEnd; + ( ( ! nComboOfTheEntry -- ) ? ( ++ pEntry, nComboOfTheEntry = pEntry->m_numStaticCombos - 1 ) : 0 ) ) + { + Assert( nComboOfTheEntry < pEntry->m_numStaticCombos ); + + // Read length + int len; + pBuf->read( &len, sizeof( len ) ); + + // Length can indicate the number of skips to make + if ( len <= 0 ) + { + // remember how many static combos get skipped + g_numSkippedStaticCombos += -len; + + // then we skip as instructed + for ( int64 numSkips = - len - 1; + numSkips > 0; ) + { + if ( numSkips <= nComboOfTheEntry ) + { + nComboOfTheEntry -= numSkips; + iCombo += numSkips; + numSkips = 0; + } + else + { + numSkips -= nComboOfTheEntry + 1; + iCombo += nComboOfTheEntry + 1; + ++ pEntry; + nComboOfTheEntry = pEntry->m_numStaticCombos - 1; + } + } + + if ( iCombo < comboEnd ) + continue; + else + break; + } + + // Shader code arrived + char const *chShaderName = pEntry->m_szName; + + // If starting new shader remember shader info + if ( chLastShaderName != chShaderName ) + { + Shader_ParseShaderInfoFromCompileCommands( pEntry, siLastShaderInfo ); + + chLastShaderName = chShaderName; + g_ShaderToShaderInfo[ chLastShaderName ] = siLastShaderInfo; + } + + // Read buffer + uint8 *pCodeBuffer = StaticComboFromDictAdd( chShaderName, nComboOfTheEntry )->AllocPackedCodeBlock( len ); + + if ( pCodeBuffer ) + pBuf->read( pCodeBuffer, len ); + } +} + + +// +// A function that will wait for right Ctrl+Alt+Shift to be held down simultaneously. +// This is useful for debugging short-lived processes and gives time for debugger to +// get attached. +// +void DebugSafeWaitPoint( bool bForceWait = false ) +{ + static bool s_bDebuggerAttached = ( CommandLine()->FindParm( "-debugwait" ) == 0 ); + + if ( bForceWait ) + { + s_bDebuggerAttached = false; + } + + if ( !s_bDebuggerAttached ) + { + Msg( "Waiting for right Ctrl+Alt+Shift to continue..." ); + while ( !s_bDebuggerAttached ) + { + Msg( "." ); + Sleep(1000); + + if ( short( GetAsyncKeyState( VK_RCONTROL ) ) < 0 && + short( GetAsyncKeyState( VK_RSHIFT ) ) < 0 && + short( GetAsyncKeyState( VK_RMENU ) ) < 0 ) + { + s_bDebuggerAttached = true; + } + } + Msg( " ok.\n" ); + } +} + +// same as "system", but doesn't pop up a window +void MySystem( char const * const pCommand, CmdSink::IResponse **ppResponse ) +{ + // Trap the command in InterceptFxc + if ( InterceptFxc::TryExecuteCommand( pCommand, ppResponse ) ) + { + Sleep( 0 ); + return; + } + + unlink( "shader.o" ); + + FILE *batFp = fopen( "temp.bat", "w" ); + fprintf( batFp, "%s\n", pCommand ); + fclose( batFp ); + + STARTUPINFO si; + PROCESS_INFORMATION pi; + + ZeroMemory( &si, sizeof(si) ); + si.cb = sizeof(si); + ZeroMemory( &pi, sizeof(pi) ); + + // Start the child process. + if( !CreateProcess( NULL, // No module name (use command line). + "temp.bat", // Command line. + NULL, // Process handle not inheritable. + NULL, // Thread handle not inheritable. + FALSE, // Set handle inheritance to FALSE. + IDLE_PRIORITY_CLASS | CREATE_NO_WINDOW, // No creation flags. + NULL, // Use parent's environment block. + g_WorkerTempPath, // Use parent's starting directory. + &si, // Pointer to STARTUPINFO structure. + &pi ) // Pointer to PROCESS_INFORMATION structure. + ) + { + Error( "CreateProcess failed." ); + Assert( 0 ); + } + + // Wait until child process exits. + WaitForSingleObject( pi.hProcess, INFINITE ); + + // Close process and thread handles. + CloseHandle( pi.hProcess ); + CloseHandle( pi.hThread ); +} + +// Assemble a reply package to the master from the compiled bytecode +// return the length of the package. +size_t AssembleWorkerReplyPackage( CfgProcessor::CfgEntryInfo const *pEntry, uint64 nComboOfEntry, + MessageBuffer *pBuf ) +{ + GLOBAL_DATA_MTX_LOCK(); + CStaticCombo *pStComboRec = StaticComboFromDict( pEntry->m_szName, nComboOfEntry ); + StaticComboNodeHash_t *pByteCodeArray = g_ShaderByteCode[ pEntry->m_szName ]; + GLOBAL_DATA_MTX_UNLOCK(); + + size_t nBytesWritten = 0; + + if ( pStComboRec && pStComboRec->m_DynamicCombos.Count() ) + { + CUtlBuffer ubDynamicComboBuffer; + ubDynamicComboBuffer.SetBigEndian( g_bIsX360 ); + + pStComboRec->SortDynamicCombos(); + // iterate over all dynamic combos. + for(int i = 0 ; i < pStComboRec->m_DynamicCombos.Count(); i++ ) + { + CByteCodeBlock *pCode = pStComboRec->m_DynamicCombos[i]; + // check if we have already output an identical combo + bool bDup = false; +#if 0 + // check for duplicate bytecode. actually doesn't save much because bzip does a good + // job compressing dupes. + for( int j = 0; j < i; j++ ) + { + if ( + ( pCode->m_nCRC32 == pStComboRec->m_DynamicCombos[j]->m_nCRC32 ) && + ( pCode->m_nCodeSize == pStComboRec->m_DynamicCombos[j]->m_nCodeSize ) && + ( memcmp( pCode->m_ByteCode, pStComboRec->m_DynamicCombos[i]->m_ByteCode, pCode->m_nCodeSize ) == 0 ) + ) // identical bytecode? + { + bDup = true; + OutputDynamicComboDup( &nBytesWritten, &ubDynamicComboBuffer, + pBuf, pCode->m_nComboID, + pStComboRec->m_DynamicCombos[j]->m_nComboID ); + } + } +#endif + if ( ! bDup ) + OutputDynamicCombo( &nBytesWritten, &ubDynamicComboBuffer, + pBuf, pCode->m_nComboID, + pCode->m_nCodeSize, pCode->m_ByteCode ); + } + FlushCombos( &nBytesWritten, &ubDynamicComboBuffer, pBuf ); + } + + // Time to limit amount of prints + static float s_fLastInfoTime = 0; + float fCurTime = ( float ) Plat_FloatTime(); + + GLOBAL_DATA_MTX_LOCK(); + if ( pStComboRec ) + pByteCodeArray->DeleteByKey( nComboOfEntry ); + if( fabs( fCurTime - s_fLastInfoTime ) > 1.f ) + { + Msg( "\rCompiling %s [ %2llu remaining ] ... \r", + pEntry->m_szName, nComboOfEntry ); + s_fLastInfoTime = fCurTime; + } + GLOBAL_DATA_MTX_UNLOCK(); + + return nBytesWritten; +} + +// Copy a reply package to the master from the compiled bytecode +// return the length of the data copied. +size_t CopyWorkerReplyPackage( CfgProcessor::CfgEntryInfo const *pEntry, uint64 nComboOfEntry, + MessageBuffer *pBuf, int nSkipsSoFar ) +{ + GLOBAL_DATA_MTX_LOCK(); + CStaticCombo *pStComboRec = StaticComboFromDict( pEntry->m_szName, nComboOfEntry ); + StaticComboNodeHash_t *pByteCodeArray = g_ShaderByteCode[ pEntry->m_szName ]; // Get a static combo pointer + GLOBAL_DATA_MTX_UNLOCK(); + + int len = pStComboRec ? pStComboRec->m_abPackedCode.GetLength() : NULL; + + if ( len ) + { + if ( nSkipsSoFar ) + { + pBuf->write( &nSkipsSoFar, sizeof( nSkipsSoFar ) ); + } + + pBuf->write( &len, sizeof( len ) ); + if ( len ) + pBuf->write( pStComboRec->m_abPackedCode.GetData(), len ); + + } + + if ( pStComboRec ) + { + GLOBAL_DATA_MTX_LOCK(); + pByteCodeArray->DeleteByKey( nComboOfEntry ); + GLOBAL_DATA_MTX_UNLOCK(); + } + + return size_t( len ); +} + + + +template < typename TMutexType > +class CWorkerAccumState : public CParallelProcessorBase < CWorkerAccumState < TMutexType > > +{ + friend ThisParallelProcessorBase_t; + +private: + static bool & DisconnectState() { static bool sb = false; return sb; } + static void Special_DisconnectHandler( int procID, const char *pReason ) { DisconnectState() = true; } + +public: + explicit CWorkerAccumState( TMutexType *pMutex ) : + m_pMutex( pMutex ), m_iFirstCommand( 0 ), m_iNextCommand( 0 ), + m_iEndCommand( 0 ), m_iLastFinished( 0 ), + m_hCombo( NULL ), + m_fnOldDisconnectHandler( g_fnDisconnectHandler ), + m_autoRestoreDisconnectHandler( g_fnDisconnectHandler ) + { + DisconnectState() = false; + } + ~CWorkerAccumState() { QuitSubs(); } + + void RangeBegin( uint64 iFirstCommand, uint64 iEndCommand ); + void RangeFinished( void ); + + void ExecuteCompileCommand( CfgProcessor::ComboHandle hCombo ); + void ExecuteCompileCommandThreaded( CfgProcessor::ComboHandle hCombo ); + void HandleCommandResponse( CfgProcessor::ComboHandle hCombo, CmdSink::IResponse *pResponse ); + +public: + using ThisParallelProcessorBase_t::Run; + +public: + bool OnProcess(); + bool OnProcessST(); + +protected: + TMutexType *m_pMutex; + +protected: + struct SubProcess + { + DWORD dwIndex; + DWORD dwSvcThreadId; + uint64 iRunningCommand; + PROCESS_INFORMATION pi; + SubProcessKernelObjects *pCommObjs; + }; + CThreadLocal < SubProcess * > m_lpSubProcessInfo; + CUtlVector < SubProcess * > m_arrSubProcessInfos; + uint64 m_iFirstCommand; + uint64 m_iNextCommand; + uint64 m_iEndCommand; + + uint64 m_iLastFinished; + + CfgProcessor::ComboHandle m_hCombo; + + DisconnectHandlerFn_t m_fnOldDisconnectHandler; + CAutoPushPop< DisconnectHandlerFn_t > m_autoRestoreDisconnectHandler; + + void QuitSubs( void ); + void TryToPackageData( uint64 iCommandNumber ); + void PrepareSubProcess( SubProcess **ppSp, SubProcessKernelObjects **ppCommObjs ); +}; + +template < typename TMutexType > +void CWorkerAccumState < TMutexType > ::RangeBegin( uint64 iFirstCommand, uint64 iEndCommand ) +{ + m_iFirstCommand = iFirstCommand; + m_iNextCommand = iFirstCommand; + m_iEndCommand = iEndCommand; + m_iLastFinished = iFirstCommand; + m_hCombo = NULL; + CfgProcessor::Combo_GetNext( m_iNextCommand, m_hCombo, m_iEndCommand ); + + g_fnDisconnectHandler = Special_DisconnectHandler; + + // Notify all connected sub-processes that the master is still alive + for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k ) + { + if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] ) + { + SubProcessKernelObjects_Memory shrmem( pSp->pCommObjs ); + if ( void *pvMemory = shrmem.Lock() ) + { + strcpy( ( char * ) pvMemory, "keepalive" ); + shrmem.Unlock(); + } + } + } +} + +template < typename TMutexType > +void CWorkerAccumState < TMutexType > ::RangeFinished( void ) +{ + if( !DisconnectState() ) + { + // Finish packaging data + TryToPackageData( m_iEndCommand - 1 ); + } + else + { + // Master disconnected + QuitSubs(); + } + + g_fnDisconnectHandler = m_fnOldDisconnectHandler; +} + +template < typename TMutexType > +void CWorkerAccumState < TMutexType > ::QuitSubs( void ) +{ + CUtlVector < HANDLE > m_arrWait; + m_arrWait.EnsureCapacity( m_arrSubProcessInfos.Count() ); + + for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k ) + { + if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] ) + { + SubProcessKernelObjects_Memory shrmem( pSp->pCommObjs ); + if ( void *pvMemory = shrmem.Lock() ) + { + strcpy( ( char * ) pvMemory, "quit" ); + shrmem.Unlock(); + } + + m_arrWait.AddToTail( pSp->pi.hProcess ); + } + } + + if ( m_arrWait.Count() ) + { + DWORD dwWait = WaitForMultipleObjects( m_arrWait.Count(), m_arrWait.Base(), TRUE, 2 * 1000 ); + if ( WAIT_TIMEOUT == dwWait ) + { + Warning( "Timed out while waiting for sub-processes to shut down!\n" ); + } + } + + for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k ) + { + if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] ) + { + CloseHandle( pSp->pi.hThread ); + CloseHandle( pSp->pi.hProcess ); + + delete pSp->pCommObjs; + delete pSp; + } + } + + if ( DisconnectState() ) + Vmpi_Worker_DefaultDisconnectHandler( 0, "Master disconnected during compilation." ); +} + +template < typename TMutexType > +void CWorkerAccumState < TMutexType > ::PrepareSubProcess( SubProcess **ppSp, SubProcessKernelObjects **ppCommObjs ) +{ + SubProcess *pSp = m_lpSubProcessInfo.Get(); + SubProcessKernelObjects *pCommObjs = NULL; + + if ( pSp ) + { + pCommObjs = pSp->pCommObjs; + } + else + { + pSp = new SubProcess; + m_lpSubProcessInfo.Set( pSp ); + + pSp->dwSvcThreadId = ThreadGetCurrentId(); + + char chBaseNameBuffer[0x30]; + sprintf( chBaseNameBuffer, "SHCMPL_SUB_%08X_%08llX_%08X", pSp->dwSvcThreadId, (long long)time( NULL ), GetCurrentProcessId() ); + pCommObjs = pSp->pCommObjs = new SubProcessKernelObjects_Create( chBaseNameBuffer ); + + ZeroMemory( &pSp->pi, sizeof( pSp->pi ) ); + + STARTUPINFO si; + ZeroMemory( &si, sizeof( si ) ); + si.cb = sizeof( si ); + + char chCommandLine[0x100]; + sprintf( chCommandLine, "\"%s\\shadercompile.exe\" -subprocess %s", g_WorkerTempPath, chBaseNameBuffer ); +#ifdef _DEBUG + V_strncat( chCommandLine, " -allowdebug", sizeof( chCommandLine ) ); +#endif + BOOL bCreateResult = CreateProcess( NULL, chCommandLine, NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, g_WorkerTempPath, &si, &pSp->pi ); + ( void ) bCreateResult; + Assert( bCreateResult && "CreateProcess failed?" ); + + m_pMutex->Lock(); + pSp->dwIndex = m_arrSubProcessInfos.AddToTail( pSp ); + m_pMutex->Unlock(); + } + + if ( ppSp ) *ppSp = pSp; + if ( ppCommObjs ) *ppCommObjs = pCommObjs; +} + +template < typename TMutexType > +void CWorkerAccumState < TMutexType > ::ExecuteCompileCommandThreaded( CfgProcessor::ComboHandle hCombo ) +{ + // DebugOut( "threaded: running: \"%s\"\n", szCommand ); + + SubProcessKernelObjects *pCommObjs = NULL; + PrepareSubProcess( NULL, &pCommObjs ); + + // Execute the command + SubProcessKernelObjects_Memory shrmem( pCommObjs ); + + { + void *pvMemory = shrmem.Lock(); + Assert( pvMemory ); + + Combo_FormatCommand( hCombo, ( char * ) pvMemory ); + + shrmem.Unlock(); + } + + // Obtain the command response + { + void const *pvMemory = shrmem.Lock(); + Assert( pvMemory ); + + // TODO: Vitaliy :: TEMP fix: + // Usually what happens if we fail to lock here is + // when our subprocess dies and to recover we will + // attempt to restart on another worker. + if ( !pvMemory ) + // ::RaiseException( GetLastError(), EXCEPTION_NONCONTINUABLE, 0, NULL ); + TerminateProcess( GetCurrentProcess(), 1 ); + + CmdSink::IResponse *pResponse; + if ( pvMemory ) + pResponse = new CSubProcessResponse( pvMemory ); + else + pResponse = new CmdSink::CResponseError; + + HandleCommandResponse( hCombo, pResponse ); + + shrmem.Unlock(); + } +} + +template < typename TMutexType > +void CWorkerAccumState < TMutexType > ::ExecuteCompileCommand( CfgProcessor::ComboHandle hCombo ) +{ + CmdSink::IResponse *pResponse = NULL; + + { + char chBuffer[ 4096 ]; + Combo_FormatCommand( hCombo, chBuffer ); + + DebugOut( "running: \"%s\"\n", chBuffer ); + + MySystem( chBuffer, &pResponse ); + } + + HandleCommandResponse( hCombo, pResponse ); +} + +template < typename TMutexType > +void CWorkerAccumState < TMutexType > ::HandleCommandResponse( CfgProcessor::ComboHandle hCombo, CmdSink::IResponse *pResponse ) +{ + VMPI_HandleSocketErrors(); + + if ( !pResponse ) + pResponse = new CmdSink::CResponseFiles( "shader.o", "output.txt" ); + + // Command info + CfgProcessor::CfgEntryInfo const *pEntryInfo = Combo_GetEntryInfo( hCombo ); + uint64 iComboIndex = Combo_GetComboNum( hCombo ); + uint64 iCommandNumber = Combo_GetCommandNum( hCombo ); + + if ( pResponse->Succeeded() ) + { + GLOBAL_DATA_MTX_LOCK(); + uint64 nStComboIdx = iComboIndex / pEntryInfo->m_numDynamicCombos; + uint64 nDyComboIdx = iComboIndex - ( nStComboIdx * pEntryInfo->m_numDynamicCombos ); + StaticComboFromDictAdd( pEntryInfo->m_szName, nStComboIdx )->AddDynamicCombo( nDyComboIdx , pResponse->GetResultBuffer(), pResponse->GetResultBufferLen() ); + GLOBAL_DATA_MTX_UNLOCK(); + + } + + // Tell the master that this shader failed + if ( !pResponse->Succeeded() ) + { + GLOBAL_DATA_MTX_LOCK(); + ShaderHadErrorDispatchInt( pEntryInfo->m_szName ); + GLOBAL_DATA_MTX_UNLOCK(); + } + + // Process listing even if the shader succeeds for warnings + char const *szListing = pResponse->GetListing(); + if ( ( !g_bSuppressWarnings && szListing ) || !pResponse->Succeeded() ) + { + char chCommandNumber[50]; + sprintf( chCommandNumber, "%I64u", iCommandNumber ); + + char chUnreportedListing[0xFF]; + if ( !szListing ) + { + sprintf( chUnreportedListing, "(0): error 0000: Compiler failed without error description, latest version of fxc.exe might give a description." ); + szListing = chUnreportedListing; + } + + // Send the listing for dispatch + CUtlBinaryBlock errMsg; + errMsg.SetLength( + strlen( chCommandNumber ) + 1 + // command + newline + strlen( szListing ) + 1 + // listing + newline + 1 // null-terminator + ); + sprintf( ( char * ) errMsg.Get(), "%s\n%s\n", chCommandNumber, szListing ); + + GLOBAL_DATA_MTX_LOCK(); + ErrMsgDispatchInt( ( char * ) errMsg.Get(), pEntryInfo->m_szShaderFileName ); + GLOBAL_DATA_MTX_UNLOCK(); + } + + // Maybe zip things up + TryToPackageData( iCommandNumber ); +} + +template < typename TMutexType > +void CWorkerAccumState < TMutexType > ::TryToPackageData( uint64 iCommandNumber ) +{ + m_pMutex->Lock(); + + uint64 iFinishedByNow = iCommandNumber + 1; + + // Check if somebody is running an earlier command + for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k ) + { + if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] ) + { + if ( pSp->iRunningCommand < iCommandNumber ) + { + iFinishedByNow = 0; + break; + } + } + } + + uint64 iLastFinished = m_iLastFinished; + if ( iFinishedByNow > m_iLastFinished ) + { + m_iLastFinished = iFinishedByNow; + m_pMutex->Unlock(); + } + else + { + m_pMutex->Unlock(); + return; + } + + CfgProcessor::ComboHandle hChBegin = CfgProcessor::Combo_GetCombo( iLastFinished ); + CfgProcessor::ComboHandle hChEnd = CfgProcessor::Combo_GetCombo( iFinishedByNow ); + + Assert( hChBegin && hChEnd ); + + CfgProcessor::CfgEntryInfo const *pInfoBegin = Combo_GetEntryInfo( hChBegin ); + CfgProcessor::CfgEntryInfo const *pInfoEnd = Combo_GetEntryInfo( hChEnd ); + + uint64 nComboBegin = Combo_GetComboNum( hChBegin ) / pInfoBegin->m_numDynamicCombos; + uint64 nComboEnd = Combo_GetComboNum( hChEnd ) / pInfoEnd->m_numDynamicCombos; + + for ( ; pInfoBegin && ( + ( pInfoBegin->m_iCommandStart < pInfoEnd->m_iCommandStart ) || + ( nComboBegin > nComboEnd ) ); ) + { + // Zip this combo + MessageBuffer mbPacked; + size_t nPackedLength = AssembleWorkerReplyPackage( pInfoBegin, nComboBegin, &mbPacked ); + + if ( nPackedLength ) + { + // Packed buffer + GLOBAL_DATA_MTX_LOCK(); + uint8 *pCodeBuffer = StaticComboFromDictAdd( pInfoBegin->m_szName, + nComboBegin )->AllocPackedCodeBlock( nPackedLength ); + GLOBAL_DATA_MTX_UNLOCK(); + + if ( pCodeBuffer ) + mbPacked.read( pCodeBuffer, nPackedLength ); + } + + // Next iteration + if ( ! nComboBegin -- ) + { + Combo_Free( hChBegin ); + if ( ( hChBegin = CfgProcessor::Combo_GetCombo( pInfoBegin->m_iCommandEnd ) ) != NULL ) + { + pInfoBegin = Combo_GetEntryInfo( hChBegin ); + nComboBegin = pInfoBegin->m_numStaticCombos - 1; + } + } + } + + Combo_Free( hChBegin ); + Combo_Free( hChEnd ); +} + + +template < typename TMutexType > +bool CWorkerAccumState < TMutexType > ::OnProcess() +{ + m_pMutex->Lock(); + CfgProcessor::ComboHandle hThreadCombo = m_hCombo ? Combo_Alloc( m_hCombo ) : NULL; + m_pMutex->Unlock(); + + uint64 iThreadCommand = ~uint64(0); + + SubProcess *pSp = NULL; + PrepareSubProcess( &pSp, NULL ); + + for ( ; ; ) + { + m_pMutex->Lock(); + if ( DisconnectState() ) + Combo_Free( m_hCombo ); + + if ( m_hCombo ) + { + Combo_Assign( hThreadCombo, m_hCombo ); + pSp->iRunningCommand = Combo_GetCommandNum( hThreadCombo ); + Combo_GetNext( iThreadCommand, m_hCombo, m_iEndCommand ); + } + else + { + Combo_Free( hThreadCombo ); + iThreadCommand = ~uint64(0); + pSp->iRunningCommand = ~uint64(0); + } + m_pMutex->Unlock(); + + if ( hThreadCombo ) + { + ExecuteCompileCommandThreaded( hThreadCombo ); + } + else + break; + } + + Combo_Free( hThreadCombo ); + return false; +} + +template < typename TMutexType > +bool CWorkerAccumState < TMutexType > ::OnProcessST() +{ + while ( m_hCombo ) + { + ExecuteCompileCommand( m_hCombo ); + + Combo_GetNext( m_iNextCommand, m_hCombo, m_iEndCommand ); + } + return false; +} + +// +// Worker_ProcessCommandRange_Singleton +// +class Worker_ProcessCommandRange_Singleton +{ +public: + static Worker_ProcessCommandRange_Singleton *& Instance() { static Worker_ProcessCommandRange_Singleton *s_ptr = NULL; return s_ptr; } + static Worker_ProcessCommandRange_Singleton * GetInstance() { Worker_ProcessCommandRange_Singleton *p = Instance(); Assert( p ); return p; } + +public: + Worker_ProcessCommandRange_Singleton() { Assert( !Instance() ); Instance() = this; Startup(); } + ~Worker_ProcessCommandRange_Singleton() { Assert( Instance() == this ); Instance() = NULL; Shutdown(); } + +public: + void ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd ); + +protected: + void Startup( void ); + void Shutdown( void ); + + // + // Multi-threaded section +protected: + struct MT { + MT() : pWorkerObj( NULL ), pThreadPool( NULL ) {} + + typedef CThreadFastMutex MultiThreadMutex_t; + MultiThreadMutex_t mtx; + + typedef CWorkerAccumState < MultiThreadMutex_t > WorkerClass_t; + WorkerClass_t *pWorkerObj; + + IThreadPool *pThreadPool; + ThreadPoolStartParams_t tpsp; + } m_MT; + + // + // Single-threaded section +protected: + struct ST { + ST() : pWorkerObj( NULL ) {} + + typedef CThreadNullMutex MultiThreadMutex_t; + MultiThreadMutex_t mtx; + + typedef CWorkerAccumState < MultiThreadMutex_t > WorkerClass_t; + WorkerClass_t *pWorkerObj; + } m_ST; +}; + +void Worker_ProcessCommandRange_Singleton::Startup( void ) +{ + bool bInitializedThreadPool = false; + CPUInformation const &cpu = *GetCPUInformation(); + + if ( cpu.m_nLogicalProcessors > 1 ) + { + // Attempt to initialize thread pool + m_MT.pThreadPool = g_pThreadPool; + if ( m_MT.pThreadPool ) + { + m_MT.tpsp.bIOThreads = false; + m_MT.tpsp.nThreads = cpu.m_nLogicalProcessors - 1; + + if ( m_MT.pThreadPool->Start( m_MT.tpsp ) ) + { + if ( m_MT.pThreadPool->NumThreads() >= 1 ) + { + // Make sure that our mutex is in multi-threaded mode + Threading::g_mtxGlobal.SetThreadedMode( Threading::eMultiThreaded ); + + m_MT.pWorkerObj = new MT::WorkerClass_t( &m_MT.mtx ); + + bInitializedThreadPool = true; + } + else + { + m_MT.pThreadPool->Stop(); + } + } + + if ( !bInitializedThreadPool ) + m_MT.pThreadPool = NULL; + } + } + + // Otherwise initialize single-threaded mode + if ( !bInitializedThreadPool ) + { + m_ST.pWorkerObj = new ST::WorkerClass_t( &m_ST.mtx ); + } +} + +void Worker_ProcessCommandRange_Singleton::Shutdown( void ) +{ + if ( m_MT.pThreadPool ) + { + if( m_MT.pWorkerObj ) + delete m_MT.pWorkerObj; + + m_MT.pThreadPool->Stop(); + m_MT.pThreadPool = NULL; + } + else + { + if ( m_ST.pWorkerObj ) + delete m_ST.pWorkerObj; + } +} + +void Worker_ProcessCommandRange_Singleton::ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd ) +{ + if ( m_MT.pThreadPool ) + { + MT::WorkerClass_t *pWorkerObj = m_MT.pWorkerObj; + + pWorkerObj->RangeBegin( shaderStart, shaderEnd ); + pWorkerObj->Run(); + pWorkerObj->RangeFinished(); + } + else + { + ST::WorkerClass_t *pWorkerObj = m_ST.pWorkerObj; + + pWorkerObj->RangeBegin( shaderStart, shaderEnd ); + pWorkerObj->OnProcessST(); + pWorkerObj->RangeFinished(); + } +} + + + +// You must process the work unit range. +void Worker_ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd ) +{ + Worker_ProcessCommandRange_Singleton::GetInstance()->ProcessCommandRange( shaderStart, shaderEnd ); +} + +// You must append data to pBuf with the work unit results. +void Worker_ProcessWorkUnitFn( int iThread, uint64 iWorkUnit, MessageBuffer *pBuf ) +{ + uint64 comboStart = iWorkUnit * g_nStaticCombosPerWorkUnit; + uint64 comboEnd = comboStart + g_nStaticCombosPerWorkUnit; + comboEnd = min( g_numStaticCombos, comboEnd ); + + // Determine the commands required to be executed: + uint64 nComboOfTheEntry = 0; + CfgProcessor::CfgEntryInfo const *pEntry = NULL; + + pEntry = GetEntryByStaticComboNum( comboEnd, &nComboOfTheEntry ); + uint64 commandEnd = pEntry->m_iCommandStart + nComboOfTheEntry * pEntry->m_numDynamicCombos; + Assert( commandEnd <= g_numCompileCommands ); + + pEntry = GetEntryByStaticComboNum( comboStart, &nComboOfTheEntry ); + uint64 commandStart = pEntry->m_iCommandStart + nComboOfTheEntry * pEntry->m_numDynamicCombos; + + // Compile all the shader combos + Worker_ProcessCommandRange( commandStart, commandEnd ); + nComboOfTheEntry = pEntry->m_numStaticCombos - 1 - nComboOfTheEntry; + + // Copy off the reply packages + int nSkipsSoFar = 0; + for ( uint64 kCombo = comboStart; kCombo < comboEnd; ++ kCombo ) + { + size_t nCpBytes = CopyWorkerReplyPackage( pEntry, nComboOfTheEntry, pBuf, nSkipsSoFar ); + if ( nCpBytes ) + nSkipsSoFar = 0; + else + -- nSkipsSoFar; + if ( nComboOfTheEntry == 0 ) + { + ++pEntry; + nComboOfTheEntry = pEntry->m_numStaticCombos; + } + nComboOfTheEntry--; + } + if ( nSkipsSoFar ) + { + pBuf->write( &nSkipsSoFar, sizeof( nSkipsSoFar ) ); + } + + ////////////////////////////////////////////////////////////////////////// + // + // Now deliver all our accumulated spew to the master + // + ////////////////////////////////////////////////////////////////////////// + + // Failed shaders + for ( int k = 0, kEnd = g_Master_ShaderHadError.GetNumStrings(); k < kEnd; ++ k ) + { + char const *szShaderName = g_Master_ShaderHadError.String( k ); + if ( !g_Master_ShaderHadError[ int_as_symid( k ) ] ) + continue; + + int const len = strlen( szShaderName ); + + CUtlBinaryBlock bb; + bb.SetLength( 1 + len + 1 ); + sprintf( ( char * ) bb.Get(), "%c%s", SHADERHADERROR_PACKETID, szShaderName ); + + VMPI_SendData( bb.Get(), bb.Length(), VMPI_MASTER_ID ); + VMPI_HandleSocketErrors(); + } + + // Compiler spew + for ( int k = 0, kEnd = g_Master_CompilerMsgInfo.GetNumStrings(); k < kEnd; ++ k ) + { + char const * const szMsg = g_Master_CompilerMsgInfo.String( k ); + CompilerMsgInfo const &cmi = g_Master_CompilerMsgInfo[ int_as_symid( k ) ]; + + char const * const szFirstCmd = cmi.GetFirstCommand(); + int const numReported = cmi.GetNumTimesReported(); + + char chNumReported[0x40]; + sprintf( chNumReported, "%d", numReported ); + + CUtlBinaryBlock bb; + bb.SetLength( 1 + strlen(szMsg) + 1 + strlen( szFirstCmd ) + 1 + strlen( chNumReported ) + 1 + 1 ); + sprintf( ( char * ) bb.Get(), "%c%s\n%s\n%s\n", ERRMSG_PACKETID, szMsg, szFirstCmd, chNumReported ); + + VMPI_SendData( bb.Get(), bb.Length(), VMPI_MASTER_ID ); + VMPI_HandleSocketErrors(); + } + + // Clean all reported msgs + g_Master_CompilerMsgInfo.Purge(); +} + + +void Shader_ParseShaderInfoFromCompileCommands( CfgProcessor::CfgEntryInfo const *pEntry, ShaderInfo_t &shaderInfo ) +{ + if ( CfgProcessor::ComboHandle hCombo = CfgProcessor::Combo_GetCombo( pEntry->m_iCommandStart ) ) + { + char cmd[ 4096 ]; + Combo_FormatCommand( hCombo, cmd ); + + { + memset( &shaderInfo, 0, sizeof( ShaderInfo_t ) ); + + const char *pCentroidMask = strstr( cmd, "/DCENTROIDMASK=" ); + const char *pFlags = strstr( cmd, "/DFLAGS=0x" ); + const char *pShaderModel = strstr( cmd, "/DSHADER_MODEL_" ); + + if( !pCentroidMask || !pFlags || !pShaderModel ) + { + Assert( !"!pCentroidMask || !pFlags || !pShaderModel" ); + return; + } + + sscanf( pCentroidMask + strlen( "/DCENTROIDMASK=" ), "%u", &shaderInfo.m_CentroidMask ); + sscanf( pFlags + strlen( "/DFLAGS=0x" ), "%x", &shaderInfo.m_Flags ); + + // Copy shader model + pShaderModel += strlen( "/DSHADER_MODEL_" ); + for ( char *pszSm = shaderInfo.m_szShaderModel, * const pszEnd = pszSm + sizeof( shaderInfo.m_szShaderModel ) - 1; + pszSm < pszEnd ; ++ pszSm ) + { + char &rchLastChar = (*pszSm = *pShaderModel ++); + if ( !rchLastChar || + V_isspace( rchLastChar ) || + '=' == rchLastChar ) + { + rchLastChar = 0; + break; + } + } + + shaderInfo.m_nShaderCombo = 0; + shaderInfo.m_nTotalShaderCombos = pEntry->m_numCombos; + shaderInfo.m_nDynamicCombos = pEntry->m_numDynamicCombos; + shaderInfo.m_nStaticCombo = 0; + + shaderInfo.m_pShaderName = pEntry->m_szName; + shaderInfo.m_pShaderSrc = pEntry->m_szShaderFileName; + } + + Combo_Free( hCombo ); + } +} + + + + +void Worker_GetLocalCopyOfShaders( void ) +{ + // Create virtual files for all of the stuff that we need to compile the shader + // make sure and prefix the file name so that it doesn't find it locally. + + char filename[1024]; + sprintf( filename, "%s\\uniquefilestocopy.txt", g_pShaderPath ); + + CUtlInplaceBuffer bffr( 0, 0, CUtlBuffer::TEXT_BUFFER ); + if( !g_pFileSystem->ReadFile( filename, NULL, bffr ) ) + { + fprintf( stderr, "Can't open uniquefilestocopy.txt!\n" ); + exit( -1 ); + } + + while( char *pszLineToCopy = bffr.InplaceGetLinePtr() ) + { + V_MakeAbsolutePath( filename, sizeof( filename ), pszLineToCopy, g_pShaderPath ); + + if ( g_bVerbose ) + printf( "getting local copy of shader: \"%s\" (\"%s\")\n", pszLineToCopy, filename ); + + CUtlBuffer fileBuf; + if ( !g_pFileSystem->ReadFile( filename, NULL, fileBuf ) ) + { + Warning( "Can't find \"%s\"\n", filename ); + continue; + } + + // Grab just the filename. + char justFilename[MAX_PATH]; + char *pLastSlash = max( strrchr( pszLineToCopy, '/' ), strrchr( pszLineToCopy, '\\' ) ); + if ( pLastSlash ) + Q_strncpy( justFilename, pLastSlash + 1, sizeof( justFilename ) ); + else + Q_strncpy( justFilename, pszLineToCopy, sizeof( justFilename ) ); + + sprintf( filename, "%s%s", g_WorkerTempPath, justFilename ); + if ( g_bVerbose ) + printf( "creating \"%s\"\n", filename ); + + FILE *fp3 = fopen( filename, "wb" ); + if ( !fp3 ) + { + Error( "Can't open '%s' for writing.", pszLineToCopy ); + continue; + } + + fwrite( fileBuf.Base(), 1, fileBuf.GetBytesRemaining(), fp3 ); + fclose( fp3 ); + + // SUPER EVIL, but if we don't do this, Windows will randomly nuke files of ours + // while we're running since they're in the temp path. + + static CUtlVector< FILE * > s_arrHackedFiles; + static struct X_s_arrHackedFiles { ~X_s_arrHackedFiles() { + for ( int k = 0; k < s_arrHackedFiles.Count(); ++ k ) + fclose( s_arrHackedFiles[k] ); + } } s_autoCloseHackedFiles; + + /* THIS IS THE EVIL LINE ----> */ FILE *fHack = fopen( filename, "r" ); + s_arrHackedFiles.AddToTail( fHack ); + // -- END of EVIL + } +} + +void Worker_GetLocalCopyOfBinary( const char *pFilename ) +{ + CUtlBuffer fileBuf; + char tmpFilename[MAX_PATH]; + sprintf( tmpFilename, "%s\\%s", g_ExeDir, pFilename ); + if ( g_bVerbose ) + printf( "trying to open: %s\n", tmpFilename ); + + FILE *fp = fopen( tmpFilename, "rb" ); + if( !fp ) + { + Assert( 0 ); + fprintf( stderr, "Can't open %s!\n", pFilename ); + exit( -1 ); + } + fseek( fp, 0, SEEK_END ); + int fileLen = ftell( fp ); + fseek( fp, 0, SEEK_SET ); + fileBuf.EnsureCapacity( fileLen ); + int nBytesRead = fread( fileBuf.Base(), 1, fileLen, fp ); + fclose( fp ); + fileBuf.SeekPut( CUtlBuffer::SEEK_HEAD, nBytesRead ); + + char newFilename[MAX_PATH]; + sprintf( newFilename, "%s%s", g_WorkerTempPath, pFilename ); + + FILE *fp2 = fopen( newFilename, "wb" ); + if( !fp2 ) + { + Assert( 0 ); + fprintf( stderr, "Can't open %s!\n", newFilename ); + exit( -1 ); + } + fwrite( fileBuf.Base(), 1, fileLen, fp2 ); + fclose( fp2 ); + + // SUPER EVIL, but if we don't do this, Windows will randomly nuke files of ours + // while we're running since they're in the temp path. + fopen( newFilename, "r" ); +} + +void Worker_GetLocalCopyOfBinaries( void ) +{ + Worker_GetLocalCopyOfBinary( "mysql_wrapper.dll" ); // This is necessary so VMPI doesn't run in SDK mode. + Worker_GetLocalCopyOfBinary( "vstdlib.dll" ); + Worker_GetLocalCopyOfBinary( "tier0.dll" ); +} + +void Shared_ParseListOfCompileCommands( void ) +{ +// double tt_start = Plat_FloatTime(); + + char fileListFileName[1024]; + sprintf( fileListFileName, "%s\\filelist.txt", g_pShaderPath ); + + CUtlInplaceBuffer bffr( 0, 0, CUtlInplaceBuffer::TEXT_BUFFER ); + if( !g_pFileSystem->ReadFile( fileListFileName, NULL, bffr) ) + { + DebugOut( "Can't open %s!\n", fileListFileName ); + fprintf( stderr, "Can't open %s!\n", fileListFileName ); + exit( -1 ); + } + + CfgProcessor::ReadConfiguration( &bffr ); + CfgProcessor::DescribeConfiguration( g_arrCompileEntries ); + + for ( CfgProcessor::CfgEntryInfo const *pInfo = g_arrCompileEntries.Get(); + pInfo && pInfo->m_szName; ++ pInfo ) + { + ++ g_numShaders; + g_numStaticCombos += pInfo->m_numStaticCombos; + g_numCompileCommands = pInfo->m_iCommandEnd; + } + +// double tt_end = Plat_FloatTime(); + + Msg( "\rCompiling %s commands. \r", PrettyPrintNumber( g_numCompileCommands ) ); +} + +void SetupExeDir( int argc, char **argv ) +{ + strcpy( g_ExeDir, argv[0] ); + Q_StripFilename( g_ExeDir ); + + if ( g_ExeDir[0] == 0 ) + { + Q_strncpy( g_ExeDir, ".\\", sizeof( g_ExeDir ) ); + } + + Q_FixSlashes( g_ExeDir ); +} + +void SetupPaths( int argc, char **argv ) +{ + GetTempPath( sizeof( g_WorkerTempPath ), g_WorkerTempPath ); + + strcat( g_WorkerTempPath, "shadercompiletemp\\" ); + char tmp[MAX_PATH]; + sprintf( tmp, "rd /s /q \"%s\"", g_WorkerTempPath ); + system( tmp ); + _mkdir( g_WorkerTempPath ); +// printf( "g_WorkerTempPath: \"%s\"\n", g_WorkerTempPath ); + + CommandLine()->CreateCmdLine( argc, argv ); + g_pShaderPath = CommandLine()->ParmValue( "-shaderpath", "" ); + + g_bVerbose = CommandLine()->FindParm("-verbose") != 0; +} + +void SetupDebugFile( void ) +{ +#ifdef DEBUGFP + const char *pComputerName = getenv( "COMPUTERNAME" ); + char filename[MAX_PATH]; + sprintf( filename, "\\\\fileserver\\user\\gary\\debug\\%s.txt", pComputerName ); + g_WorkerDebugFp = fopen( filename, "w" ); + Assert( g_WorkerDebugFp ); + DebugOut( "opened debug file\n" ); +#endif +} + +void CompileShaders_NoVMPI() +{ + Worker_ProcessCommandRange_Singleton pcr; + + // + // We will iterate on the cfg entries and process them + // + for ( CfgProcessor::CfgEntryInfo const *pEntry = g_arrCompileEntries.Get(); + pEntry && pEntry->m_szName; ++ pEntry ) + { + // + // Stick the shader info + // + ShaderInfo_t siLastShaderInfo; + memset( &siLastShaderInfo, 0, sizeof( siLastShaderInfo ) ); + + Shader_ParseShaderInfoFromCompileCommands( pEntry, siLastShaderInfo ); + + g_ShaderToShaderInfo[ pEntry->m_szName ] = siLastShaderInfo; + + // + // Compile stuff + // + Worker_ProcessCommandRange( pEntry->m_iCommandStart, pEntry->m_iCommandEnd ); + + // + // Now when the whole shader is finished we can write it + // + char const *szShaderToWrite = pEntry->m_szName; + g_numCommandsCompleted = g_numCompileCommands; + WriteShaderFiles( szShaderToWrite ); + g_numCommandsCompleted = pEntry->m_iCommandEnd; + } + + Msg( "\r \r" ); +} + + +class CDistributeShaderCompileMaster : public IWorkUnitDistributorCallbacks +{ +public: + CDistributeShaderCompileMaster( void ); + ~CDistributeShaderCompileMaster( void ); + +public: + virtual void OnWorkUnitsCompleted( uint64 numWorkUnits ); + +private: + void ThreadProc( void ); + friend DWORD WINAPI CDistributeShaderCompileMaster::ThreadProcAdapter( LPVOID pvArg ); + static DWORD WINAPI ThreadProcAdapter( LPVOID pvArg ) { reinterpret_cast< CDistributeShaderCompileMaster * >( pvArg )->ThreadProc(); return 0; } + +private: + HANDLE m_hThread; + HANDLE m_hEvent; + CThreadFastMutex m_mtx; + BOOL m_bRunning; + +private: + CfgProcessor::CfgEntryInfo const *m_pAnalyzeShaders; + CUtlVector< char const * > m_arrShaderNamesToWrite; +}; + +CDistributeShaderCompileMaster::CDistributeShaderCompileMaster( void ) : + m_hThread( NULL ), + m_hEvent( NULL ), + m_bRunning( TRUE ) +{ + m_hEvent = CreateEvent( NULL, FALSE, FALSE, NULL ); + m_hThread = CreateThread( NULL, 0, ThreadProcAdapter, reinterpret_cast< LPVOID >(this), 0, NULL ); + + m_pAnalyzeShaders = g_arrCompileEntries.Get(); +} + +CDistributeShaderCompileMaster::~CDistributeShaderCompileMaster( void ) +{ + m_bRunning = FALSE; + + SetEvent( m_hEvent ); + WaitForSingleObject( m_hThread, INFINITE ); + + CloseHandle( m_hThread ); + CloseHandle( m_hEvent ); +} + +void CDistributeShaderCompileMaster::OnWorkUnitsCompleted( uint64 numWorkUnits ) +{ + // Make sure that our mutex is in multi-threaded mode + Threading::g_mtxGlobal.SetThreadedMode( Threading::eMultiThreaded ); + + // Figure out how many commands have completed based on work units + g_numCompletedStaticCombos = numWorkUnits * g_nStaticCombosPerWorkUnit; + uint64 numStaticCombosOfTheEntry = 0; + CfgProcessor::CfgEntryInfo const *pEntry = GetEntryByStaticComboNum( g_numCompletedStaticCombos, &numStaticCombosOfTheEntry ); + g_numCommandsCompleted = pEntry->m_iCommandStart + numStaticCombosOfTheEntry * pEntry->m_numDynamicCombos; + + // Iterate over the shaders yet to be written and see if we can queue them + for ( ; m_pAnalyzeShaders->m_szName && + m_pAnalyzeShaders->m_iCommandEnd <= g_numCommandsCompleted; + ++ m_pAnalyzeShaders + ) + { + m_mtx.Lock(); + m_arrShaderNamesToWrite.AddToTail( m_pAnalyzeShaders->m_szName ); + SetEvent( m_hEvent ); + m_mtx.Unlock(); + } +} + +void CDistributeShaderCompileMaster::ThreadProc( void ) +{ + for ( ; m_bRunning; ) + { + WaitForSingleObject( m_hEvent, INFINITE ); + + // Do a pump of shaders to write + for ( int numShadersWritten = 0; /* forever */ ; ++ numShadersWritten ) + { + m_mtx.Lock(); + char const * szShaderToWrite = NULL; + if ( m_arrShaderNamesToWrite.Count() > numShadersWritten ) + szShaderToWrite = m_arrShaderNamesToWrite[ numShadersWritten ]; + else + m_arrShaderNamesToWrite.RemoveAll(); + m_mtx.Unlock(); + + if ( !szShaderToWrite ) + break; + + // We have the shader to write asynchronously + WriteShaderFiles( szShaderToWrite ); + } + } +} + +int ShaderCompile_Main( int argc, char* argv[] ) +{ + InstallSpewFunction(); + g_bSuppressPrintfOutput = false; + g_flStartTime = Plat_FloatTime(); + + SetupDebugFile(); + numthreads = 1; // managed specifically in Worker_ProcessCommandRange_Singleton::Startup + + /* + Special section of code implementing "-subprocess" flag + */ + if ( int iSubprocess = CommandLine()->FindParm( "-subprocess" ) ) + { + char const *szSubProcessData = CommandLine()->GetParm( 1 + iSubprocess ); + return ShaderCompile_Subprocess_Main( szSubProcessData ); + } + + // This needs to get called before VMPI is setup because in SDK mode, VMPI will change the args around. + SetupExeDir( argc, argv ); + + g_bIsX360 = CommandLine()->FindParm( "-x360" ) != 0; + // g_bSuppressWarnings = g_bIsX360; + + bool bShouldUseVMPI = ( CommandLine()->FindParm( "-nompi" ) == 0 ); + if ( bShouldUseVMPI ) + { + // Master, start accepting connections. + // Worker, make a connection. + DebugOut( "Before VMPI_Init\n" ); + g_bSuppressPrintfOutput = true; + VMPIRunMode mode = VMPI_RUN_NETWORKED; + if ( !VMPI_Init( argc, argv, "dependency_info_shadercompile.txt", MyDisconnectHandler, mode ) ) + { + g_bSuppressPrintfOutput = false; + DebugOut( "MPI_Init failed.\n" ); + Error( "MPI_Init failed." ); + } + + extern void VMPI_SetWorkUnitsPartitionSize( int numWusToDeal ); + VMPI_SetWorkUnitsPartitionSize( 32 ); + } + + SetupPaths( argc, argv ); + + g_bSuppressPrintfOutput = false; + DebugOut( "After VMPI_Init\n" ); + + // Setting up the minidump handlers + if ( bShouldUseVMPI && !g_bMPIMaster ) + SetupToolsMinidumpHandler( VMPI_ExceptionFilter ); + else + SetupDefaultToolsMinidumpHandler(); + + if ( CommandLine()->FindParm( "-game" ) == 0 ) + { + // Used with filesystem_stdio.dll + FileSystem_Init( NULL, 0, FS_INIT_COMPATIBILITY_MODE ); + } + else + { + // SDK uses this since it only has filesystem_steam.dll. + FileSystem_Init( NULL, 0, FS_INIT_FULL ); + } + + DebugOut( "After VMPI_FileSystem_Init\n" ); + Shared_ParseListOfCompileCommands(); + DebugOut( "After Shared_ParseListOfCompileCommands\n" ); + + if ( bShouldUseVMPI ) + { + // Partition combos + g_nStaticCombosPerWorkUnit = 0; + if ( g_numStaticCombos ) + { + if ( g_numStaticCombos <= 1024 ) + g_nStaticCombosPerWorkUnit = 1; + else if ( g_numStaticCombos > 1024 * 10 ) + g_nStaticCombosPerWorkUnit = 10; + else + g_nStaticCombosPerWorkUnit = g_numStaticCombos / 1024; + } + + uint64 nWorkUnits; + if( g_nStaticCombosPerWorkUnit == 0 ) + { + nWorkUnits = 1; + g_nStaticCombosPerWorkUnit = g_numStaticCombos; + } + else + { + nWorkUnits = g_numStaticCombos / g_nStaticCombosPerWorkUnit + 1; + } + + DebugOut( "Before conditional\n" ); + if ( g_bMPIMaster ) + { + // Send all of the workers the complete list of work to do. + DebugOut( "Before STARTWORK_PACKETID\n" ); + + char packetID = STARTWORK_PACKETID; + VMPI_SendData( &packetID, sizeof( packetID ), VMPI_PERSISTENT ); + + // Compile master distribution tracker + CDistributeShaderCompileMaster dscm; + g_pDistributeWorkCallbacks = &dscm; + + { + char chCommands[50], chStaticCombos[50], chNumWorkUnits[50]; + sprintf( chCommands, "%s", PrettyPrintNumber( g_numCompileCommands ) ); + sprintf( chStaticCombos, "%s", PrettyPrintNumber( g_numStaticCombos ) ); + sprintf( chNumWorkUnits, "%s", PrettyPrintNumber( nWorkUnits ) ); + Msg( "\rCompiling %s commands in %s work units.\n", chCommands, chNumWorkUnits ); + } + + // nWorkUnits is how many work units. . .1000 is good. + // The work unit number impies which combo to do. + DebugOut( "Before DistributeWork\n" ); + DistributeWork( nWorkUnits, WORKUNIT_PACKETID, NULL, Master_ReceiveWorkUnitFn ); + + g_pDistributeWorkCallbacks = NULL; + } + else + { + // wait until we get a packet from the master to start doing stuff. + MessageBuffer buf; + DebugOut( "Before VMPI_DispatchUntil\n" ); + while ( !g_bGotStartWorkPacket ) + { + VMPI_DispatchNextMessage(); + } + DebugOut( "after VMPI_DispatchUntil\n" ); + + DebugOut( "Before Worker_GetLocalCopyOfShaders\n" ); + Worker_GetLocalCopyOfShaders(); + DebugOut( "Before Worker_GetLocalCopyOfBinaries\n" ); + Worker_GetLocalCopyOfBinaries(); + + DebugOut( "Before _chdir\n" ); + _chdir( g_WorkerTempPath ); + + // nWorkUnits is how many work units. . .1000 is good. + // The work unit number impies which combo to do. + DebugOut( "Before DistributeWork\n" ); + + // Allows calling into ProcessCommandRange inside the worker function + { + Worker_ProcessCommandRange_Singleton pcr; + DistributeWork( nWorkUnits, WORKUNIT_PACKETID, Worker_ProcessWorkUnitFn, NULL ); + } + } + + g_bSuppressPrintfOutput = true; + g_bSuppressPrintfOutput = false; + } + else // no VMPI + { + Worker_GetLocalCopyOfShaders(); + Worker_GetLocalCopyOfBinaries(); + _chdir( g_WorkerTempPath ); + + { + char chCommands[50], chStaticCombos[50]; + sprintf( chCommands, "%s", PrettyPrintNumber( g_numCompileCommands ) ); + sprintf( chStaticCombos, "%s", PrettyPrintNumber( g_numStaticCombos ) ); + Msg( "\rCompiling %s commands in %s static combos.\n", chCommands, chStaticCombos ); + } + CompileShaders_NoVMPI(); + } + + Msg( "\r \r" ); + if ( g_bMPIMaster || !bShouldUseVMPI ) + { + char str[ 4096 ]; + + // Write everything that succeeded + int nStrings = g_ShaderByteCode.GetNumStrings(); + for( int i = 0; i < nStrings; i++ ) + { + WriteShaderFiles( g_ShaderByteCode.String(i) ); + } + + // Write all the errors + ////////////////////////////////////////////////////////////////////////// + // + // Now deliver all our accumulated spew to the output + // + ////////////////////////////////////////////////////////////////////////// + + bool bValveVerboseComboErrors = ( getenv( "VALVE_VERBOSE_COMBO_ERRORS" ) && + atoi( getenv( "VALVE_VERBOSE_COMBO_ERRORS" ) ) ) ? true : false; + + // Compiler spew + for ( int k = 0, kEnd = g_Master_CompilerMsgInfo.GetNumStrings(); k < kEnd; ++ k ) + { + char const * const szMsg = g_Master_CompilerMsgInfo.String( k ); + CompilerMsgInfo const &cmi = g_Master_CompilerMsgInfo[ int_as_symid( k ) ]; + + char const * const szFirstCmd = cmi.GetFirstCommand(); + int const numReported = cmi.GetNumTimesReported(); + + uint64 iFirstCommand = _strtoui64( szFirstCmd, NULL, 10 ); + CfgProcessor::ComboHandle hCombo = NULL; + CfgProcessor::CfgEntryInfo const *pComboEntryInfo = NULL; + if ( CfgProcessor::Combo_GetNext( iFirstCommand, hCombo, g_numCompileCommands ) ) + { + Combo_FormatCommand( hCombo, str ); + pComboEntryInfo = Combo_GetEntryInfo( hCombo ); + Combo_Free( hCombo ); + } + else + { + sprintf( str, "cmd # %s", szFirstCmd ); + } + + + Msg( "\n%s\n", szMsg ); + Msg( " Reported %d time(s), example command:\n", numReported); + + if ( bValveVerboseComboErrors ) + { + Msg( " Verbose Description:\n" ); + if ( pComboEntryInfo ) + { + Msg( " Src File: %s\n", pComboEntryInfo->m_szShaderFileName ); + Msg( " Tgt File: %s\n", pComboEntryInfo->m_szName ); + } + + // Between /DSHADERCOMBO= and /Dmain + char const *pBegin = strstr( str, "/DSHADERCOMBO=" ); + char const *pEnd = strstr( str, "/Dmain" ); + if ( pBegin ) + { + pBegin += strlen( "/DSHADERCOMBO=" ) ; + char const *pSpace = strchr( pBegin, ' ' ); + if ( pSpace ) + Msg( " Combo # : %.*s\n", ( pSpace - pBegin ), pBegin ); + } + + if ( !pEnd ) + pEnd = str + strlen( str ); + while ( pBegin && *pBegin && !V_isspace( *pBegin ) ) + ++ pBegin; + while ( pBegin && *pBegin && V_isspace( *pBegin ) ) + ++ pBegin; + + // Now parse all combo defines in [pBegin, pEnd] + while ( pBegin && *pBegin && ( pBegin < pEnd ) ) + { + char const *pDefine = strstr( pBegin, "/D" ); + if ( !pDefine || pDefine >= pEnd ) + break; + + char const *pEqSign = strchr( pDefine, '=' ); + if ( !pEqSign || pEqSign >= pEnd ) + break; + + char const *pSpace = strchr( pEqSign, ' ' ); + if ( !pSpace || pSpace >= pEnd ) + pSpace = pEnd; + + pBegin = pSpace; + + Msg( " %.*s %.*s\n", + ( pSpace - pEqSign - 1 ), pEqSign + 1, + ( pEqSign - pDefine - 2 ), pDefine + 2 ); + } + } + Msg( " %s\n", str ); + } + + // Failed shaders summary + for ( int k = 0, kEnd = g_Master_ShaderHadError.GetNumStrings(); k < kEnd; ++ k ) + { + char const *szShaderName = g_Master_ShaderHadError.String( k ); + if ( !g_Master_ShaderHadError[ int_as_symid( k ) ] ) + continue; + + Msg( "FAILED: %s\n", szShaderName ); + } + + // + // End + // + double end = Plat_FloatTime(); + + GetHourMinuteSecondsString( (int)( end - g_flStartTime ), str, sizeof( str ) ); + DebugOut( "%s elapsed\n", str ); + DebugOut( "Precise timing = %.5f\n", ( end - g_flStartTime ) ); + + if ( bShouldUseVMPI ) + { + VMPI_FileSystem_Term(); + DebugOut( "Before VMPI_Finalize\n" ); + VMPI_Finalize(); + } + } + + return g_Master_ShaderHadError.GetNumStrings(); +} + +class CShaderCompileDLL : public IShaderCompileDLL +{ + int main( int argc, char **argv ); +}; + +int CShaderCompileDLL::main( int argc, char **argv ) +{ + return ShaderCompile_Main( argc, argv ); +} + +EXPOSE_SINGLE_INTERFACE( CShaderCompileDLL, IShaderCompileDLL, SHADER_COMPILE_INTERFACE_VERSION ); + + +class CLaunchableDLL : public ILaunchableDLL +{ + int main( int argc, char **argv ) + { + return ShaderCompile_Main( argc, argv ); + } +}; + +EXPOSE_SINGLE_INTERFACE( CLaunchableDLL, ILaunchableDLL, LAUNCHABLE_DLL_INTERFACE_VERSION ); -- cgit v1.2.3