1HEADmaster

1 files changed, 1258 insertions, 0 deletions

diff --git a/materialsystem/shaderapidx9/textureheap.cpp b/materialsystem/shaderapidx9/textureheap.cpp
new file mode 100644
index 0000000..b2c9a66
--- /dev/null
+++ b/materialsystem/shaderapidx9/textureheap.cpp
@@ -0,0 +1,1258 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose:
+//
+//=============================================================================
+
+#include "tier1/mempool.h"
+#include "tier1/convar.h"
+#include "tier1/utlmap.h"
+#include "shaderapidx8.h"
+#include "texturedx8.h"
+#include "textureheap.h"
+#include "shaderapidx8_global.h"
+
+#include "tier0/memdbgon.h"
+
+#define USE_STANDARD_ALLOCATOR
+#ifdef USE_STANDARD_ALLOCATOR
+#define UseStandardAllocator() (true)
+#elif !defined(_RETAIL)
+bool g_bUseStandardAllocator = false;
+bool UseStandardAllocator()
+{
+	static bool bReadCommandLine;
+	if ( !bReadCommandLine )
+	{
+		bReadCommandLine = true;
+		const char *pStr = Plat_GetCommandLine();
+		if ( pStr )
+		{
+			char tempStr[512];
+			Q_strncpy( tempStr, pStr, sizeof( tempStr ) - 1 );
+			tempStr[ sizeof( tempStr ) - 1 ] = 0;
+			_strlwr( tempStr );
+
+			if ( strstr( tempStr, "-notextureheap" ) )
+				g_bUseStandardAllocator = true;
+		}
+	}
+	return g_bUseStandardAllocator;
+}
+#else
+#define UseStandardAllocator() (false)
+#endif
+
+#if !defined( _RELEASE ) && !defined( _RETAIL )
+#define StrongAssert( expr )	if ( (expr) ) ; else { DebuggerBreak(); }
+#else
+#define StrongAssert( expr )	((void)0)
+#endif
+
+//-----------------------------------------------------------------------------
+// Get Texture HW base
+//-----------------------------------------------------------------------------
+void *GetD3DTextureBasePtr( IDirect3DBaseTexture* pTex )
+{
+	// assumes base and mips are contiguous
+	return (void *)( (unsigned int)pTex->Format.BaseAddress << 12 );
+}
+
+class CD3DTextureAllocator
+{
+public:
+	static void *Alloc( int bytes )
+	{
+		DWORD attributes = MAKE_XALLOC_ATTRIBUTES( 
+								0, 
+								false, 
+								TRUE, 
+								FALSE, 
+								eXALLOCAllocatorId_D3D,
+								XALLOC_PHYSICAL_ALIGNMENT_4K, 
+								XALLOC_MEMPROTECT_WRITECOMBINE, 
+								FALSE,
+								XALLOC_MEMTYPE_PHYSICAL );
+		m_nTotalAllocations++;
+		m_nTotalSize += AlignValue( bytes, 4096 );
+		return XMemAlloc( bytes, attributes );
+	}
+
+	static void Free( void *p )
+	{
+		DWORD attributes = MAKE_XALLOC_ATTRIBUTES( 
+								0, 
+								false, 
+								TRUE, 
+								FALSE, 
+								eXALLOCAllocatorId_D3D,
+								XALLOC_PHYSICAL_ALIGNMENT_4K, 
+								XALLOC_MEMPROTECT_WRITECOMBINE, 
+								FALSE,
+								XALLOC_MEMTYPE_PHYSICAL );
+		m_nTotalAllocations--;
+		m_nTotalSize -= XMemSize( p, attributes );
+		XMemFree( p, attributes );
+	}
+
+	static int GetAllocations()
+	{
+		return m_nTotalAllocations;
+	}
+
+	static int GetSize()
+	{
+		return m_nTotalSize;
+	}
+
+	static int	m_nTotalSize;
+	static int	m_nTotalAllocations;
+};
+
+int CD3DTextureAllocator::m_nTotalSize;
+int CD3DTextureAllocator::m_nTotalAllocations;
+
+enum TextureAllocator_t
+{
+	TA_DEFAULT,
+	TA_MIXED,
+	TA_UNKNOWN,
+};
+
+struct THBaseInfo
+{
+	TextureAllocator_t	m_fAllocator;
+	int					m_TextureSize;	// stored for delayed allocations
+};
+
+struct THInfo_t : public THBaseInfo
+{
+	// Mixed heap info
+	int nLogicalBytes;
+	int nBytes;
+	bool bFree:1;
+	bool bNonTexture:1;
+
+	THInfo_t *pPrev, *pNext;
+};
+
+struct THFreeBlock_t
+{
+	THInfo_t heapInfo;
+	THFreeBlock_t *pPrevFree, *pNextFree;
+};
+
+class CXboxTexture : public IDirect3DTexture, public THInfo_t
+{
+public:
+	CXboxTexture() 
+	  : bImmobile(false)
+	{
+	}
+
+	bool bImmobile;
+	bool CanRelocate()	{ return ( !bImmobile && !IsBusy() ); }
+};
+
+class CXboxCubeTexture : public IDirect3DCubeTexture, public THBaseInfo
+{
+};
+
+class CXboxVolumeTexture : public IDirect3DVolumeTexture, public THBaseInfo
+{
+};
+
+
+void SetD3DTextureImmobile( IDirect3DBaseTexture *pTexture, bool bImmobile )
+{
+	if ( pTexture->GetType() == D3DRTYPE_TEXTURE )
+	{
+		(( CXboxTexture *)pTexture)->bImmobile = bImmobile;
+	}
+}
+
+CXboxTexture *GetTexture( THInfo_t *pInfo )
+{
+	if ( !pInfo->bFree && !pInfo->bNonTexture )
+	{
+		return (CXboxTexture *)((byte *)pInfo - offsetof( CXboxTexture, m_fAllocator ));
+	}
+	return NULL;
+}
+
+inline THFreeBlock_t *GetFreeBlock( THInfo_t *pInfo )
+{
+	if ( pInfo->bFree )
+	{
+		return (THFreeBlock_t *)((byte *)pInfo - offsetof( THFreeBlock_t, heapInfo ));
+	}
+	return NULL;
+}
+
+class CMixedTextureHeap
+{
+	enum
+	{
+		SIZE_ALIGNMENT = XBOX_HDD_SECTORSIZE,
+		MIN_BLOCK_SIZE = 1024,
+	};
+public:
+
+	CMixedTextureHeap() :
+		m_nLogicalBytes( 0 ),
+		m_nActualBytes( 0 ),
+		m_nAllocs( 0 ),
+		m_nOldBytes( 0 ),
+		m_nNonTextureAllocs( 0 ),
+		m_nBytesTotal( 0 ),
+		m_pBase( NULL ),
+		m_pFirstFree( NULL )
+	{
+	}
+
+	void Init()
+	{
+		extern ConVar mat_texturecachesize;
+		MEM_ALLOC_CREDIT_("CMixedTextureHeap");
+
+		m_nBytesTotal = ( mat_texturecachesize.GetInt() * 1024 * 1024 );
+#if 0
+		m_nBytesTotal = AlignValue( m_nBytesTotal, SIZE_ALIGNMENT );
+		m_pBase = CD3DTextureAllocator::Alloc( m_nBytesTotal );
+#else
+		m_nBytesTotal = AlignValue( m_nBytesTotal, 16*1024*1024 );
+		m_pBase = XPhysicalAlloc( m_nBytesTotal, MAXULONG_PTR, 4096, PAGE_READWRITE | PAGE_WRITECOMBINE | MEM_16MB_PAGES );
+#endif
+		m_pFirstFree = (THFreeBlock_t *)m_pBase;
+
+
+		m_pFirstFree->heapInfo.bFree = true;
+		m_pFirstFree->heapInfo.bNonTexture = false;
+		m_pFirstFree->heapInfo.nBytes = m_nBytesTotal;
+		m_pFirstFree->heapInfo.pNext = NULL;
+		m_pFirstFree->heapInfo.pPrev = NULL;
+		m_pFirstFree->pNextFree = NULL;
+		m_pFirstFree->pPrevFree = NULL;
+
+		m_pLastFree = m_pFirstFree;
+	}
+
+	void *Alloc( int bytes, THInfo_t *pInfo, bool bNonTexture = false )
+	{
+		pInfo->nBytes = AlignValue( bytes, SIZE_ALIGNMENT );
+
+		if ( !m_pBase )
+		{
+			Init();
+		}
+
+		if ( bNonTexture && m_nNonTextureAllocs == 0 )
+		{
+			Compact();
+		}
+
+		void *p = FindBlock( pInfo );
+
+		if ( !p )
+		{
+			p = ExpandToFindBlock( pInfo );
+		}
+
+		if ( p )
+		{
+			pInfo->nLogicalBytes = bytes;
+			pInfo->bNonTexture = bNonTexture;
+			m_nLogicalBytes += bytes;
+			if ( !IsRetail() )
+			{
+				m_nOldBytes += AlignValue( bytes, 4096 );
+			}
+			m_nActualBytes += pInfo->nBytes;
+			m_nAllocs++;
+
+			if ( bNonTexture )
+			{
+				m_nNonTextureAllocs++;
+			}
+		}
+		return p;
+	}
+
+	void Free( void *p, THInfo_t *pInfo )
+	{
+		if ( !p )
+		{
+			return;
+		}
+
+		if ( !IsRetail() )
+		{
+			m_nOldBytes -= AlignValue( pInfo->nLogicalBytes, 4096 );
+		}
+
+		if ( pInfo->bNonTexture )
+		{
+			m_nNonTextureAllocs--;
+		}
+
+		m_nLogicalBytes -= pInfo->nLogicalBytes;
+		m_nAllocs--;
+		m_nActualBytes -= pInfo->nBytes;
+
+		THFreeBlock_t *pFree = (THFreeBlock_t *)p;
+		pFree->heapInfo = *pInfo;
+		pFree->heapInfo.bFree = true;
+
+		AddToBlocksList( &pFree->heapInfo, pFree->heapInfo.pPrev, pFree->heapInfo.pNext );
+
+		pFree = MergeLeft( pFree );
+		pFree = MergeRight( pFree );
+
+		AddToFreeList( pFree );
+
+		if ( pInfo->bNonTexture && m_nNonTextureAllocs == 0 )
+		{
+			Compact();
+		}
+	}
+
+	int Size( void *p, THInfo_t *pInfo )
+	{
+		return AlignValue( pInfo->nBytes, SIZE_ALIGNMENT );
+	}
+
+	bool IsOwner( void *p )
+	{
+		return ( m_pBase && p >= m_pBase && p < (byte *)m_pBase + m_nBytesTotal );
+	}
+
+	//-----------------------------------------------------
+
+	void *FindBlock( THInfo_t *pInfo )
+	{
+		THFreeBlock_t *pCurrent = m_pFirstFree;
+
+		int nBytesDesired = pInfo->nBytes;
+
+		// Find the first block big enough to hold, then split it if appropriate
+		while ( pCurrent && pCurrent->heapInfo.nBytes < nBytesDesired )
+		{
+			pCurrent = pCurrent->pNextFree;
+		}
+
+		if ( pCurrent )
+		{
+			return ClaimBlock( pCurrent, pInfo );
+		}
+
+		return NULL;
+	}
+
+	void AddToFreeList( THFreeBlock_t *pFreeBlock )
+	{
+		if ( !IsRetail() )
+		{
+			pFreeBlock->heapInfo.nLogicalBytes = 0;
+		}
+
+		if ( m_pFirstFree )
+		{
+			THFreeBlock_t *pPrev = NULL;
+			THFreeBlock_t *pNext = m_pFirstFree;
+
+			int nBytes = pFreeBlock->heapInfo.nBytes;
+
+			while ( pNext && pNext->heapInfo.nBytes < nBytes )
+			{
+				pPrev = pNext;
+				pNext = pNext->pNextFree;
+			}
+
+			pFreeBlock->pPrevFree = pPrev;
+			pFreeBlock->pNextFree = pNext;
+
+			if ( pPrev )
+			{
+				pPrev->pNextFree = pFreeBlock;
+			}
+			else
+			{
+				m_pFirstFree = pFreeBlock;
+			}
+
+			if ( pNext )
+			{
+				pNext->pPrevFree = pFreeBlock;
+			}
+			else
+			{
+				m_pLastFree = pFreeBlock;
+			}
+		}
+		else
+		{
+			pFreeBlock->pPrevFree = pFreeBlock->pNextFree = NULL;
+			m_pLastFree = m_pFirstFree = pFreeBlock;
+		}
+	}
+
+	void RemoveFromFreeList( THFreeBlock_t *pFreeBlock )
+	{
+		if ( m_pFirstFree == pFreeBlock )
+		{
+			m_pFirstFree = m_pFirstFree->pNextFree;
+		}
+		else if ( pFreeBlock->pPrevFree )
+		{
+			pFreeBlock->pPrevFree->pNextFree = pFreeBlock->pNextFree;
+		}
+
+		if ( m_pLastFree == pFreeBlock )
+		{
+			m_pLastFree = pFreeBlock->pPrevFree;
+		}
+		else if ( pFreeBlock->pNextFree )
+		{
+			pFreeBlock->pNextFree->pPrevFree = pFreeBlock->pPrevFree;
+		}
+
+		pFreeBlock->pPrevFree = pFreeBlock->pNextFree = NULL;
+	}
+
+	THFreeBlock_t *GetLastFree()
+	{
+		return m_pLastFree;
+	}
+
+	void AddToBlocksList( THInfo_t *pBlock, THInfo_t *pPrev, THInfo_t *pNext )
+	{
+		if ( pPrev )
+		{
+			pPrev->pNext = pBlock;
+		}
+
+		if ( pNext)
+		{
+			pNext->pPrev = pBlock;
+		}
+
+		pBlock->pPrev = pPrev;
+		pBlock->pNext = pNext;
+	}
+
+	void RemoveFromBlocksList( THInfo_t *pBlock )
+	{
+		if ( pBlock->pPrev )
+		{
+			pBlock->pPrev->pNext = pBlock->pNext;
+		}
+
+		if ( pBlock->pNext )
+		{
+			pBlock->pNext->pPrev = pBlock->pPrev;
+		}
+	}
+
+	//-----------------------------------------------------
+
+	void *ClaimBlock( THFreeBlock_t *pFreeBlock, THInfo_t *pInfo )
+	{
+		RemoveFromFreeList( pFreeBlock );
+
+		int nBytesDesired = pInfo->nBytes;
+		int nBytesRemainder = pFreeBlock->heapInfo.nBytes - nBytesDesired;
+		*pInfo = pFreeBlock->heapInfo;
+		pInfo->bFree = false;
+		pInfo->bNonTexture = false;
+		if ( nBytesRemainder >= MIN_BLOCK_SIZE )
+		{
+			pInfo->nBytes = nBytesDesired;
+
+			THFreeBlock_t *pRemainder = (THFreeBlock_t *)(((byte *)(pFreeBlock)) + nBytesDesired);
+			pRemainder->heapInfo.bFree = true;
+			pRemainder->heapInfo.nBytes = nBytesRemainder;
+
+			AddToBlocksList( &pRemainder->heapInfo, pInfo, pInfo->pNext );
+			AddToFreeList( pRemainder );
+		}
+		AddToBlocksList( pInfo, pInfo->pPrev, pInfo->pNext );
+		return pFreeBlock;
+	}
+
+	THFreeBlock_t *MergeLeft( THFreeBlock_t *pFree )
+	{
+		THInfo_t *pPrev = pFree->heapInfo.pPrev;
+		if ( pPrev && pPrev->bFree )
+		{
+			pPrev->nBytes += pFree->heapInfo.nBytes;
+			RemoveFromBlocksList( &pFree->heapInfo );
+			pFree = GetFreeBlock( pPrev );
+			RemoveFromFreeList( pFree );
+		}
+		return pFree;
+	}
+
+	THFreeBlock_t *MergeRight( THFreeBlock_t *pFree )
+	{
+		THInfo_t *pNext = pFree->heapInfo.pNext;
+		if ( pNext && pNext->bFree )
+		{
+			pFree->heapInfo.nBytes += pNext->nBytes;
+			RemoveFromBlocksList( pNext );
+			RemoveFromFreeList( GetFreeBlock( pNext ) );
+		}
+		return pFree;
+	}
+
+	//-----------------------------------------------------
+
+	bool GetExpansionList( THFreeBlock_t *pFreeBlock, THInfo_t **ppStart, THInfo_t **ppEnd, int depth = 1 )
+	{
+		THInfo_t *pStart;
+		THInfo_t *pEnd;
+		int i;
+
+		pStart = &pFreeBlock->heapInfo;
+		pEnd = &pFreeBlock->heapInfo;
+
+		if ( m_nNonTextureAllocs > 0 )
+		{
+			return false;
+		}
+
+		// Walk backwards to start of expansion
+		i = depth;
+		while ( i > 0 && pStart->pPrev)
+		{
+			THInfo_t *pScan = pStart->pPrev;
+
+			while ( i > 0 && pScan && !pScan->bFree && GetTexture( pScan )->CanRelocate() )
+			{
+				pScan = pScan->pPrev;
+				i--;
+			}
+
+			if ( !pScan || !pScan->bFree )
+			{
+				break;
+			}
+
+			pStart = pScan;
+		}
+
+		// Walk forwards to start of expansion
+		i = depth;
+		while ( i > 0 && pEnd->pNext)
+		{
+			THInfo_t *pScan = pStart->pNext;
+
+			while ( i > 0 && pScan && !pScan->bFree && GetTexture( pScan )->CanRelocate() )
+			{
+				pScan = pScan->pNext;
+				i--;
+			}
+
+			if ( !pScan || !pScan->bFree )
+			{
+				break;
+			}
+
+			pEnd = pScan;
+		}
+
+		*ppStart = pStart;
+		*ppEnd = pEnd;
+
+		return ( pStart != pEnd );
+	}
+
+	THFreeBlock_t *CompactExpansionList( THInfo_t *pStart, THInfo_t *pEnd )
+	{
+// X360TBD:
+Assert( 0 );
+return NULL;
+#if 0
+#ifdef TH_PARANOID
+		Validate();
+#endif
+		StrongAssert( pStart->bFree );
+		StrongAssert( pEnd->bFree );
+		byte *pNextBlock = (byte *)pStart;
+
+		THInfo_t *pTextureBlock = pStart;
+		THInfo_t *pLastBlock = pStart->pPrev;
+
+		while ( pTextureBlock != pEnd )
+		{
+			CXboxTexture *pTexture = GetTexture( pTextureBlock );
+			// If it's a texture, move it and thread it on. Otherwise, discard it
+			if ( pTexture )
+			{
+				void *pTextureBits = GetD3DTextureBasePtr( pTexture );
+				int nBytes = pTextureBlock->nBytes;
+
+				if ( pNextBlock + nBytes <= pTextureBits)
+				{
+					memcpy( pNextBlock, pTextureBits, nBytes );
+				}
+				else
+				{
+					memmove( pNextBlock, pTextureBits, nBytes );
+				}
+
+				pTexture->Data = 0;
+				pTexture->Register( pNextBlock );
+
+				pNextBlock += nBytes;
+				if ( pLastBlock)
+				{
+					pLastBlock->pNext = pTextureBlock;
+				}
+				pTextureBlock->pPrev = pLastBlock;
+				pLastBlock = pTextureBlock;
+			}
+			else
+			{
+				StrongAssert( pTextureBlock->bFree );
+				RemoveFromFreeList( GetFreeBlock( pTextureBlock ) );
+			}
+			pTextureBlock = pTextureBlock->pNext;
+		}
+
+		RemoveFromFreeList( GetFreeBlock( pEnd ) );
+
+		// Make a new block and fix up the block lists
+		THFreeBlock_t *pFreeBlock = (THFreeBlock_t *)pNextBlock;
+		pFreeBlock->heapInfo.pPrev = pLastBlock;
+		pLastBlock->pNext = &pFreeBlock->heapInfo;
+		pFreeBlock->heapInfo.pNext = pEnd->pNext;
+		if ( pEnd->pNext )
+		{
+			pEnd->pNext->pPrev = &pFreeBlock->heapInfo;
+		}
+		pFreeBlock->heapInfo.bFree = true;
+		pFreeBlock->heapInfo.nBytes = ( (byte *)pEnd - pNextBlock ) + pEnd->nBytes;
+		
+		AddToFreeList( pFreeBlock );
+
+#ifdef TH_PARANOID
+		Validate();
+#endif
+		return pFreeBlock;
+#endif
+	}
+
+	THFreeBlock_t *ExpandBlock( THFreeBlock_t *pFreeBlock, int depth = 1 )
+	{
+		THInfo_t *pStart;
+		THInfo_t *pEnd;
+
+		if ( GetExpansionList( pFreeBlock, &pStart, &pEnd, depth ) )
+		{
+			return CompactExpansionList( pStart, pEnd );
+		}
+
+		return pFreeBlock;
+	}
+
+	THFreeBlock_t *ExpandBlockToFit( THFreeBlock_t *pFreeBlock, unsigned bytes )
+	{
+		if ( pFreeBlock )
+		{
+			THInfo_t *pStart;
+			THInfo_t *pEnd;
+
+			if ( GetExpansionList( pFreeBlock, &pStart, &pEnd, 2 ) )
+			{
+				unsigned sum = 0;
+				THInfo_t *pCurrent = pStart;
+				while( pCurrent != pEnd->pNext ) 
+				{
+					if ( pCurrent->bFree )
+					{
+						sum += pCurrent->nBytes;
+					}
+					pCurrent = pCurrent->pNext;
+				}
+
+				if ( sum >= bytes )
+				{
+					pFreeBlock = CompactExpansionList( pStart, pEnd );
+				}
+			}
+		}
+
+		return pFreeBlock;
+	}
+
+	void *ExpandToFindBlock( THInfo_t *pInfo )
+	{
+		THFreeBlock_t *pFreeBlock = ExpandBlockToFit( GetLastFree(), pInfo->nBytes );
+		if ( pFreeBlock && pFreeBlock->heapInfo.nBytes >= pInfo->nBytes )
+		{
+			return ClaimBlock( pFreeBlock, pInfo );
+		}
+		return NULL;
+	}
+
+	void Compact()
+	{
+		if ( m_nNonTextureAllocs > 0 )
+		{
+			return;
+		}
+
+		for (;;)
+		{
+			THFreeBlock_t *pCurrent = m_pFirstFree;
+			THFreeBlock_t *pNew;
+			while ( pCurrent )
+			{
+				int nBytesOld = pCurrent->heapInfo.nBytes;
+				pNew = ExpandBlock( pCurrent, 999999 );
+
+				if ( pNew != pCurrent || pNew->heapInfo.nBytes != nBytesOld )
+				{
+#ifdef TH_PARANOID
+					Validate();
+#endif
+					break;
+				}
+
+#ifdef TH_PARANOID
+				pNew = ExpandBlock( pCurrent, 999999 );
+				StrongAssert( pNew == pCurrent && pNew->heapInfo.nBytes == nBytesOld );
+#endif
+
+				pCurrent = pCurrent->pNextFree;
+			}
+
+			if ( !pCurrent )
+			{
+				break;
+			}
+		}
+	}
+
+	void Validate()
+	{
+		if ( !m_pFirstFree )
+		{
+			return;
+		}
+
+		if ( m_nNonTextureAllocs > 0 )
+		{
+			return;
+		}
+
+		THInfo_t *pLast = NULL;
+		THInfo_t *pInfo = &m_pFirstFree->heapInfo;
+
+		while ( pInfo->pPrev )
+		{
+			pInfo = pInfo->pPrev;
+		}
+
+		void *pNextExpectedAddress = m_pBase;
+
+		while ( pInfo )
+		{
+			byte *pCurrentAddress = (byte *)(( pInfo->bFree ) ? GetFreeBlock( pInfo ) : GetD3DTextureBasePtr( GetTexture( pInfo ) ) );
+			StrongAssert( pCurrentAddress == pNextExpectedAddress );
+			StrongAssert( pInfo->pPrev == pLast );
+			pNextExpectedAddress = pCurrentAddress + pInfo->nBytes;
+			pLast = pInfo;
+			pInfo = pInfo->pNext;
+		}
+
+		THFreeBlock_t *pFree = m_pFirstFree;
+		THFreeBlock_t *pLastFree = NULL;
+		int nBytesHeap = XPhysicalSize( m_pBase );
+
+		while ( pFree )
+		{
+			StrongAssert( pFree->pPrevFree == pLastFree );
+			StrongAssert( (void *)pFree >= m_pBase && (void *)pFree < (byte *)m_pBase + nBytesHeap );
+			StrongAssert( !pFree->pPrevFree || ( (void *)pFree->pPrevFree >= m_pBase && (void *)pFree->pPrevFree < (byte *)m_pBase + nBytesHeap ) );
+			StrongAssert( !pFree->pNextFree || ( (void *)pFree->pNextFree >= m_pBase && (void *)pFree->pNextFree < (byte *)m_pBase + nBytesHeap ) );
+			StrongAssert( !pFree->pPrevFree || pFree->pPrevFree->heapInfo.nBytes <= pFree->heapInfo.nBytes );
+			pLastFree = pFree;
+			pFree = pFree->pNextFree;
+		}
+	}
+
+	//-----------------------------------------------------
+
+	THFreeBlock_t *m_pFirstFree;
+	THFreeBlock_t *m_pLastFree;
+	void *m_pBase;
+
+	int m_nLogicalBytes;
+	int m_nActualBytes;
+	int m_nAllocs;
+	int m_nOldBytes;
+	int m_nNonTextureAllocs;
+	int m_nBytesTotal;
+};
+
+//-----------------------------------------------------------------------------
+
+inline TextureAllocator_t GetTextureAllocator( IDirect3DBaseTexture9 *pTexture )
+{
+	return ( pTexture->GetType() == D3DRTYPE_CUBETEXTURE ) ? (( CXboxCubeTexture *)pTexture)->m_fAllocator : (( CXboxTexture *)pTexture)->m_fAllocator;
+}
+
+//-----------------------------------------------------------------------------
+
+CMixedTextureHeap g_MixedTextureHeap;
+
+CON_COMMAND( mat_texture_heap_stats, "" )
+{
+	if ( UseStandardAllocator() )
+	{
+		Msg( "Texture heap stats: (Standard Allocator)\n" );
+		Msg( "Allocations:%d Size:%d\n", CD3DTextureAllocator::GetAllocations(), CD3DTextureAllocator::GetSize() );
+	}
+	else
+	{
+		Msg( "Texture heap stats:\n" );
+		Msg( "  Mixed textures:    %dk/%dk allocated in %d textures\n", g_MixedTextureHeap.m_nLogicalBytes/1024, g_MixedTextureHeap.m_nActualBytes/1024, g_MixedTextureHeap.m_nAllocs );
+		float oldFootprint = g_MixedTextureHeap.m_nOldBytes;
+		float newFootprint = g_MixedTextureHeap.m_nActualBytes;
+		Msg( "\n  Old: %.3fmb, New: %.3fmb\n", oldFootprint / (1024.0*1024.0), newFootprint / (1024.0*1024.0) );
+	}
+}
+
+CON_COMMAND( mat_texture_heap_compact, "" )
+{
+	Msg( "Validating texture heap...\n" );
+	g_MixedTextureHeap.Validate();
+	Msg( "Compacting texture heap...\n" );
+	unsigned oldLargest = ( g_MixedTextureHeap.GetLastFree() ) ? g_MixedTextureHeap.GetLastFree()->heapInfo.nBytes : 0;
+	g_MixedTextureHeap.Compact();
+	unsigned newLargest = ( g_MixedTextureHeap.GetLastFree() ) ? g_MixedTextureHeap.GetLastFree()->heapInfo.nBytes : 0;
+
+	Msg( "\n  Old largest block: %.3fk, New largest block: %.3fk\n\n", oldLargest / 1024.0, newLargest / 1024.0 );
+
+	Msg( "Validating texture heap...\n" );
+	g_MixedTextureHeap.Validate();
+	Msg( "Done.\n" );
+}
+
+//-----------------------------------------------------------------------------
+// Nasty back doors
+//-----------------------------------------------------------------------------
+
+void CompactTextureHeap()
+{
+	unsigned oldLargest = ( g_MixedTextureHeap.GetLastFree() ) ? g_MixedTextureHeap.GetLastFree()->heapInfo.nBytes : 0;
+	g_MixedTextureHeap.Compact();
+	unsigned newLargest = ( g_MixedTextureHeap.GetLastFree() ) ? g_MixedTextureHeap.GetLastFree()->heapInfo.nBytes : 0;
+
+	DevMsg( "Compacted texture heap. Old largest block: %.3fk, New largest block: %.3fk\n", oldLargest / 1024.0, newLargest / 1024.0 );
+}
+
+CTextureHeap g_TextureHeap;
+
+//-----------------------------------------------------------------------------
+// Build and alloc a texture resource
+//-----------------------------------------------------------------------------
+IDirect3DTexture *CTextureHeap::AllocTexture( int width, int height, int levels, DWORD usage, D3DFORMAT d3dFormat, bool bFallback, bool bNoD3DMemory )
+{
+	CXboxTexture* pD3DTexture = new CXboxTexture;
+
+	// create a texture with contiguous mips and packed tails
+	DWORD dwTextureSize = XGSetTextureHeaderEx( 
+		width, 
+		height, 
+		levels, 
+		usage, 
+		d3dFormat, 
+		0, 
+		0, 
+		0, 
+		XGHEADER_CONTIGUOUS_MIP_OFFSET, 
+		0, 
+		pD3DTexture, 
+		NULL, 
+		NULL );
+
+	// based on "Xbox 360 Texture Storage"
+	// can truncate the terminal tile using packed tails
+	// the terminal tile must be at 32x32 or 16x16 packed
+	if ( width == height && levels != 0 )
+	{
+		int terminalWidth = width >> (levels - 1);
+		if ( d3dFormat == D3DFMT_DXT1 )
+		{
+			if ( terminalWidth <= 32 ) 
+			{
+				dwTextureSize -= 4*1024;
+			}
+		}
+		else if ( d3dFormat == D3DFMT_DXT5 ) 
+		{
+			if ( terminalWidth == 32 )
+			{
+				dwTextureSize -= 8*1024;
+			}
+			else if ( terminalWidth <= 16 )
+			{
+				dwTextureSize -= 12*1024;
+			}
+		}
+	}
+
+	pD3DTexture->m_TextureSize = dwTextureSize;
+
+	if ( !bFallback && bNoD3DMemory )
+	{
+		pD3DTexture->m_fAllocator = TA_UNKNOWN;
+		return pD3DTexture;
+	}
+
+	void *pBuffer;
+	if ( UseStandardAllocator() )
+	{
+		MEM_ALLOC_CREDIT_( __FILE__ ": Standard D3D" );
+		pBuffer = CD3DTextureAllocator::Alloc( dwTextureSize );
+		pD3DTexture->m_fAllocator = TA_DEFAULT;
+	}
+	else
+	{
+		MEM_ALLOC_CREDIT_( __FILE__ ": Mixed texture" );
+		pBuffer = g_MixedTextureHeap.Alloc( dwTextureSize, pD3DTexture );
+		if ( pBuffer )
+		{
+			pD3DTexture->m_fAllocator = TA_MIXED;
+		}
+		else
+		{
+			g_MixedTextureHeap.Compact();
+			pBuffer = g_MixedTextureHeap.Alloc( dwTextureSize, pD3DTexture );
+			if ( pBuffer )
+			{
+				pD3DTexture->m_fAllocator = TA_MIXED;
+			}
+			else
+			{
+				pBuffer = CD3DTextureAllocator::Alloc( dwTextureSize );
+				pD3DTexture->m_fAllocator = TA_DEFAULT;
+			}
+		}
+	}
+
+	if ( !pBuffer )
+	{
+		delete pD3DTexture;
+		return NULL;
+	}
+
+	XGOffsetResourceAddress( pD3DTexture, pBuffer );
+
+	return pD3DTexture;
+}
+
+//-----------------------------------------------------------------------------
+// Build and alloc a cube texture resource
+//-----------------------------------------------------------------------------
+IDirect3DCubeTexture *CTextureHeap::AllocCubeTexture( int width, int levels, DWORD usage, D3DFORMAT d3dFormat, bool bFallback, bool bNoD3DMemory )
+{
+	CXboxCubeTexture* pD3DCubeTexture = new CXboxCubeTexture;
+
+	// create a cube texture with contiguous mips and packed tails
+	DWORD dwTextureSize = XGSetCubeTextureHeaderEx( 
+		width,  
+		levels, 
+		usage, 
+		d3dFormat, 
+		0, 
+		0, 
+		0,
+		XGHEADER_CONTIGUOUS_MIP_OFFSET, 
+		pD3DCubeTexture, 
+		NULL, 
+		NULL );
+	pD3DCubeTexture->m_TextureSize = dwTextureSize;
+
+	if ( !bFallback && bNoD3DMemory )
+	{
+		pD3DCubeTexture->m_fAllocator = TA_UNKNOWN;
+		return pD3DCubeTexture;
+	}
+
+	void *pBits;
+	if ( UseStandardAllocator() )
+	{
+		MEM_ALLOC_CREDIT_( __FILE__ ": Cubemap standard D3D" );
+		pBits = CD3DTextureAllocator::Alloc( dwTextureSize );
+		pD3DCubeTexture->m_fAllocator = TA_DEFAULT;
+	}
+	else
+	{
+		// @todo: switch to texture heap
+		MEM_ALLOC_CREDIT_( __FILE__ ": Odd sized cubemap textures" );
+		// Really only happens with environment map
+		pBits = CD3DTextureAllocator::Alloc( dwTextureSize );
+		pD3DCubeTexture->m_fAllocator = TA_DEFAULT;
+	}
+
+	if ( !pBits )
+	{
+		delete pD3DCubeTexture;
+		return NULL;
+	}
+
+	XGOffsetResourceAddress( pD3DCubeTexture, pBits );
+
+	return pD3DCubeTexture;
+}
+
+//-----------------------------------------------------------------------------
+// Allocate an Volume Texture
+//-----------------------------------------------------------------------------
+IDirect3DVolumeTexture *CTextureHeap::AllocVolumeTexture( int width, int height, int depth, int levels, DWORD usage, D3DFORMAT d3dFormat )
+{
+	CXboxVolumeTexture *pD3DVolumeTexture = new CXboxVolumeTexture;
+
+	// create a cube texture with contiguous mips and packed tails
+	DWORD dwTextureSize = XGSetVolumeTextureHeaderEx( 
+		width,
+		height, 
+		depth,
+		levels, 
+		usage, 
+		d3dFormat, 
+		0, 
+		0, 
+		0,
+		XGHEADER_CONTIGUOUS_MIP_OFFSET, 
+		pD3DVolumeTexture, 
+		NULL, 
+		NULL );
+
+	void *pBits;
+	
+	MEM_ALLOC_CREDIT_( __FILE__ ": Volume standard D3D" );
+
+	pBits = CD3DTextureAllocator::Alloc( dwTextureSize );
+	pD3DVolumeTexture->m_fAllocator = TA_DEFAULT;
+	pD3DVolumeTexture->m_TextureSize = dwTextureSize;
+
+	if ( !pBits )
+	{
+		delete pD3DVolumeTexture;
+		return NULL;
+	}
+
+	XGOffsetResourceAddress( pD3DVolumeTexture, pBits );
+
+	return pD3DVolumeTexture; 
+}
+
+//-----------------------------------------------------------------------------
+// Get current backbuffer multisample type (used in AllocRenderTargetSurface() )
+//-----------------------------------------------------------------------------
+D3DMULTISAMPLE_TYPE CTextureHeap::GetBackBufferMultiSampleType()
+{
+	int backWidth, backHeight;
+	ShaderAPI()->GetBackBufferDimensions( backWidth, backHeight );
+
+	// 2xMSAA at 640x480 and 848x480 are the only supported multisample mode on 360 (2xMSAA for 720p would
+	// use predicated tiling, which would require a rewrite of *all* our render target code)
+	// FIXME: shuffle the EDRAM surfaces to allow 4xMSAA for standard def
+	//        (they would overlap & trash each other with the current allocation scheme)
+	D3DMULTISAMPLE_TYPE backBufferMultiSampleType = g_pShaderDevice->IsAAEnabled() ? D3DMULTISAMPLE_2_SAMPLES : D3DMULTISAMPLE_NONE;
+	Assert( ( g_pShaderDevice->IsAAEnabled() == false ) || (backHeight == 480) );
+
+	return backBufferMultiSampleType;
+}
+
+//-----------------------------------------------------------------------------
+// Allocate an EDRAM surface
+//-----------------------------------------------------------------------------
+IDirect3DSurface *CTextureHeap::AllocRenderTargetSurface( int width, int height, D3DFORMAT d3dFormat, bool bMultiSample, int base )
+{	
+	// render target surfaces don't need to exist simultaneously
+	// force their allocations to overlap at the end of back buffer and zbuffer
+	// this should leave 3MB (of 10) free assuming 1280x720 (and 5MB with 640x480@2xMSAA)
+	D3DMULTISAMPLE_TYPE backBufferMultiSampleType = GetBackBufferMultiSampleType();
+	D3DMULTISAMPLE_TYPE multiSampleType = bMultiSample ? backBufferMultiSampleType : D3DMULTISAMPLE_NONE;
+	if ( base < 0 )
+	{
+		int backWidth, backHeight;
+		ShaderAPI()->GetBackBufferDimensions( backWidth, backHeight );
+		D3DFORMAT backBufferFormat = ImageLoader::ImageFormatToD3DFormat( g_pShaderDevice->GetBackBufferFormat() );
+		base = 2*XGSurfaceSize( backWidth, backHeight, backBufferFormat, backBufferMultiSampleType );
+	}
+
+	D3DSURFACE_PARAMETERS surfParameters;
+	surfParameters.Base = base;
+	surfParameters.ColorExpBias = 0;
+
+	if ( ( d3dFormat == D3DFMT_D24FS8 ) || ( d3dFormat == D3DFMT_D24S8 ) || ( d3dFormat == D3DFMT_D16 ) )
+	{
+		surfParameters.HierarchicalZBase = 0;
+		if ( ( surfParameters.HierarchicalZBase + XGHierarchicalZSize( width, height, multiSampleType ) ) > GPU_HIERARCHICAL_Z_TILES )
+		{
+			// overflow, can't hold the tiles so disable
+			surfParameters.HierarchicalZBase = 0xFFFFFFFF; 
+		}
+	}
+	else
+	{
+		// not using
+		surfParameters.HierarchicalZBase = 0xFFFFFFFF;
+	}	
+
+	HRESULT hr;
+	IDirect3DSurface9 *pSurface = NULL;
+	hr = Dx9Device()->CreateRenderTarget( width, height, d3dFormat, multiSampleType, 0, FALSE, &pSurface, &surfParameters );
+	Assert( !FAILED( hr ) );
+
+	return pSurface;
+}
+
+//-----------------------------------------------------------------------------
+// Perform the real d3d allocation, returns true if succesful, false otherwise.
+// Only valid for a texture created with no d3d bits, otherwise no-op.
+//-----------------------------------------------------------------------------
+bool CTextureHeap::AllocD3DMemory( IDirect3DBaseTexture *pD3DTexture )
+{
+	if ( !pD3DTexture )
+	{
+		return false;
+	}
+
+	if ( pD3DTexture->GetType() == D3DRTYPE_SURFACE )
+	{
+		// there are no d3d bits for a surface
+		return false;
+	}
+
+	void *pBits = GetD3DTextureBasePtr( pD3DTexture );
+	if ( pBits )
+	{
+		// already have d3d bits
+		return true;
+	}
+
+	if ( pD3DTexture->GetType() == D3DRTYPE_TEXTURE )
+	{
+		MEM_ALLOC_CREDIT_( __FILE__ ": Standard D3D" );
+		pBits = CD3DTextureAllocator::Alloc( ((CXboxTexture *)pD3DTexture)->m_TextureSize );
+		((CXboxTexture *)pD3DTexture)->m_fAllocator = TA_DEFAULT;
+		XGOffsetResourceAddress( (CXboxTexture *)pD3DTexture, pBits );
+		return true;
+	}
+	else if ( pD3DTexture->GetType() == D3DRTYPE_CUBETEXTURE )
+	{
+		MEM_ALLOC_CREDIT_( __FILE__ ": Cubemap standard D3D" );
+		pBits = CD3DTextureAllocator::Alloc( ((CXboxCubeTexture *)pD3DTexture)->m_TextureSize );
+		((CXboxCubeTexture *)pD3DTexture)->m_fAllocator = TA_DEFAULT;
+		XGOffsetResourceAddress( (CXboxCubeTexture *)pD3DTexture, pBits );
+		return true;
+	}
+
+	return false;
+}
+
+//-----------------------------------------------------------------------------
+// Release the allocated store
+//-----------------------------------------------------------------------------
+void CTextureHeap::FreeTexture( IDirect3DBaseTexture *pD3DTexture )
+{
+	if ( !pD3DTexture )
+	{
+		return;
+	}
+
+	if ( pD3DTexture->GetType() == D3DRTYPE_SURFACE )
+	{
+		// texture heap doesn't own render target surfaces
+		// allow callers to call through for less higher level detection
+		int ref = ((IDirect3DSurface*)pD3DTexture)->Release();
+		Assert( ref == 0 );
+		ref = ref; // Quiet "unused variable" warning in release
+		return;
+	}
+	else
+	{
+		byte *pBits = (byte *)GetD3DTextureBasePtr( pD3DTexture );
+		if ( pBits )
+		{
+			switch ( GetTextureAllocator( pD3DTexture ) )
+			{
+			case TA_DEFAULT:
+				CD3DTextureAllocator::Free( pBits );
+				break;
+
+			case TA_MIXED:
+				g_MixedTextureHeap.Free( pBits, ((CXboxTexture *)pD3DTexture) );
+				break;
+			}
+		}
+	}
+
+	if ( pD3DTexture->GetType() == D3DRTYPE_TEXTURE )
+	{
+		delete (CXboxTexture *)pD3DTexture;
+	}
+	else if ( pD3DTexture->GetType() == D3DRTYPE_VOLUMETEXTURE )
+	{
+		delete (CXboxVolumeTexture *)pD3DTexture;
+	}
+	else if ( pD3DTexture->GetType() == D3DRTYPE_CUBETEXTURE )
+	{
+		delete (CXboxCubeTexture *)pD3DTexture;
+	}
+}
+
+//-----------------------------------------------------------------------------
+// Returns the allocated footprint
+//-----------------------------------------------------------------------------
+int	CTextureHeap::GetSize( IDirect3DBaseTexture *pD3DTexture )
+{
+	if( pD3DTexture == NULL )
+		return 0;
+
+	if ( pD3DTexture->GetType() == D3DRTYPE_SURFACE )
+	{
+		D3DSURFACE_DESC surfaceDesc;
+		HRESULT hr = ((IDirect3DSurface*)pD3DTexture)->GetDesc( &surfaceDesc );
+		Assert( !FAILED( hr ) );
+		hr = hr; // Quiet "unused variable" warning in release
+
+		int size = ImageLoader::GetMemRequired( 
+			surfaceDesc.Width,
+			surfaceDesc.Height,
+			0,
+			ImageLoader::D3DFormatToImageFormat( surfaceDesc.Format ),
+			false );
+
+		return size;
+	}
+	else if ( pD3DTexture->GetType() == D3DRTYPE_TEXTURE )
+	{
+		return ((CXboxTexture *)pD3DTexture)->m_TextureSize;
+	}
+	else if ( pD3DTexture->GetType() == D3DRTYPE_CUBETEXTURE )
+	{
+		return ((CXboxCubeTexture *)pD3DTexture)->m_TextureSize;
+	}
+	else if ( pD3DTexture->GetType() == D3DRTYPE_VOLUMETEXTURE )
+	{
+		return ((CXboxVolumeTexture *)pD3DTexture)->m_TextureSize;
+	}
+
+	return 0;
+}
+
+//-----------------------------------------------------------------------------
+// Crunch the pools
+//-----------------------------------------------------------------------------
+void CTextureHeap::Compact()
+{
+	g_MixedTextureHeap.Compact();
+}