1HEADmaster

1 files changed, 1190 insertions, 0 deletions

diff --git a/public/gcsdk/sdocache.h b/public/gcsdk/sdocache.h
new file mode 100644
index 0000000..850aef3
--- /dev/null
+++ b/public/gcsdk/sdocache.h
@@ -0,0 +1,1190 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: Serialized Digital Object caching and manipulation
+//
+//=============================================================================
+
+#ifndef SBOCACHE_H
+#define SBOCACHE_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "tier1/utlhashmaplarge.h"
+#include "tier1/utlqueue.h"
+#include "tier1/utlvector.h"
+#include <algorithm>
+
+namespace GCSDK
+{
+// Call to register SDOs. All SDO types must be registered before loaded
+#define REG_SDO( classname )	GSDOCache().RegisterSDO( classname::k_eType, #classname )
+
+// A string used to tell the difference between nil objects and actual objects in memcached
+extern const char k_rgchNilObjSerializedValue[];
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Keeps a moving average of a data set
+//-----------------------------------------------------------------------------
+template< int SAMPLES >
+class CMovingAverage
+{
+public:
+	CMovingAverage()
+	{
+		Reset();
+	}
+
+	void Reset()
+	{
+		memset( m_rglSamples, 0, sizeof( m_rglSamples ) );
+		m_cSamples = 0;
+		m_lTotal = 0;
+	}
+
+	void AddSample( int64 lSample )
+	{
+		int iIndex = m_cSamples % SAMPLES;
+		m_lTotal += ( lSample - m_rglSamples[iIndex] );
+		m_rglSamples[iIndex] = lSample;
+		m_cSamples++;
+	}
+
+	uint64 GetAveragedSample() const
+	{
+		if ( !m_cSamples )
+			return 0;
+
+		int64 iMax = (int64)MIN( m_cSamples, SAMPLES );
+		return m_lTotal / iMax;
+	}
+
+private:
+	int64 m_rglSamples[SAMPLES];
+	int64 m_lTotal;
+	uint64 m_cSamples;
+};
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Global accessor to the manager
+//-----------------------------------------------------------------------------
+class CSDOCache;
+CSDOCache &GSDOCache();
+
+
+//-----------------------------------------------------------------------------
+// Purpose: interface to a Database Backed Object
+//-----------------------------------------------------------------------------
+class ISDO
+{
+public:
+	virtual ~ISDO() {}
+
+	// Identification
+	virtual int GetType() const = 0;
+	virtual uint32 GetHashCode() const = 0;
+	virtual bool IsEqual( const ISDO *pSDO ) const = 0;
+
+	// Ref counting
+	virtual int AddRef() = 0;
+	virtual int Release() = 0;
+	virtual int GetRefCount() = 0;
+
+	// memory usage
+	virtual size_t CubBytesUsed() = 0;
+
+	// Serialization tools
+	virtual bool BReadFromBuffer( const byte *pubData, int cubData ) = 0;
+	virtual void WriteToBuffer( CUtlBuffer &memBuffer ) = 0;
+
+	// memcached batching tools
+	virtual void GetMemcachedKeyName( CFmtStr &strDest ) = 0;
+
+	// SQL loading
+	virtual bool BYldLoadFromSQL( CUtlVector<ISDO *> &vecSDOToLoad, CUtlVector<bool> &vecResults ) const = 0;
+
+	// post-load initialization (whether loaded from SQL or memcached)
+	virtual void PostLoadInit() = 0;
+
+	// comparison function for validating memcached copies vs SQL copies
+	virtual bool IsIdentical( ISDO *pSDO ) = 0;
+
+	// Returns true if this is not the actual object, but a placeholder to remember that this object
+	// doesn't actually exist
+	virtual bool BNilObject() const = 0;
+
+	// Allocs an SDO that must return true for IsEqual( this ) and BNilObject(). This is so we can
+	// cache load attempts for objects that don't exist
+	virtual ISDO *AllocNilObject() = 0;
+
+	// Creates a key name that looks like "Prefix_%u" but faster
+};
+
+
+//-----------------------------------------------------------------------------
+// Purpose: base class for a Serialized Digital Object
+//-----------------------------------------------------------------------------
+template<typename KeyType, int eSDOType, class ProtoMsg>
+class CBaseSDO : public ISDO
+{
+public:
+	typedef KeyType KeyType_t;
+	enum { k_eType = eSDOType };
+
+	CBaseSDO( const KeyType &key ) : m_Key( key ), m_nRefCount( 0 ) {}
+
+	const KeyType &GetKey() const		{ return m_Key; }
+
+	// ISDO implementation
+	virtual int AddRef();
+	virtual int Release();
+	virtual int GetRefCount();
+	virtual int GetType() const			{ return eSDOType; }
+	virtual bool BNilObject() const			{ return false; }
+	virtual uint32 GetHashCode() const;
+	virtual bool BReadFromBuffer( const byte *pubData, int cubData );
+	virtual void WriteToBuffer( CUtlBuffer &memBuffer );
+
+	// Implement this in a subclass if there's some value like 0 or -1 that's invalid so
+	// the system can know to not even attempt to load it
+	static bool BKeyValid( const KeyType_t &key ) { return true; }
+
+	// We use protobufs for all serialization
+	virtual void SerializeToProtobuf( ProtoMsg &msg ) = 0;
+	virtual bool DeserializeFromProtobuf( const ProtoMsg &msg ) = 0;
+
+	// default comparison function - override to do your own compare
+	virtual bool IsEqual( const ISDO *pSDO ) const;
+
+	// default load from SQL is no-op as not all types have permanent storage - override to create a
+	// batch load
+	virtual bool BYldLoadFromSQL( CUtlVector<ISDO *> &vecSDOToLoad, CUtlVector<bool> &vecResults ) const;
+
+	// override to do initialization after load
+	virtual void PostLoadInit() {}
+
+	// compares the serialized versions by default. Override to have more specific behavior
+	virtual bool IsIdentical( ISDO *pSDO );
+
+	// Creates a copy of the object with the same key
+	virtual ISDO *AllocNilObject();
+
+	// tools
+	bool WriteToMemcached();
+	bool DeleteFromMemcached();
+
+	// Creates a key name that looks like "Prefix_%u" but faster. Also makes sure the correct buffer size is passed
+	template <size_t prefixBufSize>
+	void CreateSimpleMemcachedName( CFmtStr &strDest, char (&rgchPrefix)[prefixBufSize], uint32 unSuffix )
+	{
+		CSDOCache::CreateSimpleMemcachedName( strDest, rgchPrefix, prefixBufSize - 1, unSuffix );
+	}
+
+private:
+	int m_nRefCount;
+	KeyType m_Key;
+};
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Represents an object we tried to load but didn't exist. We use
+//	this to cache the failure so we don't keep trying to look it up
+//-----------------------------------------------------------------------------
+template<typename KeyType, int eSDOType, class ProtoMsg>
+class CNilSDO : public CBaseSDO<KeyType, eSDOType, ProtoMsg>
+{
+public:
+	CNilSDO( const KeyType &key, const char *pchMemcachedKeyName ) : CBaseSDO<KeyType, eSDOType, ProtoMsg>( key ), m_sMemcachedKeyName( pchMemcachedKeyName ) {}
+
+	virtual bool BNilObject() const										{ return true; }
+	virtual bool BReadFromBuffer( const byte *pubData, int cubData )	{ return false; }
+	virtual void WriteToBuffer( CUtlBuffer &memBuffer )					{ memBuffer.PutString( k_rgchNilObjSerializedValue ); }
+	virtual void SerializeToProtobuf( ProtoMsg &msg )					{}
+	virtual bool DeserializeFromProtobuf( const ProtoMsg &msg )			{ return false; }
+	virtual size_t CubBytesUsed()										{ return sizeof( *this ) + m_sMemcachedKeyName.Length(); }
+	virtual void GetMemcachedKeyName( CFmtStr &strKey )					{ V_strncpy( strKey.Access(), m_sMemcachedKeyName, FMTSTR_STD_LEN ); }
+private:
+	CUtlString m_sMemcachedKeyName;
+};
+
+
+//-----------------------------------------------------------------------------
+// Purpose: references to a database-backed object
+//			maintains refcount of the object
+//-----------------------------------------------------------------------------
+template<class T>
+class CSDORef
+{
+	T *m_pSDO;
+public:
+	CSDORef() { m_pSDO = NULL; }
+	explicit CSDORef( CSDORef<T> &SDORef ) { m_pSDO = SDORef.Get(); if ( m_pSDO ) m_pSDO->AddRef(); }
+	explicit CSDORef( T *pSDO ) { m_pSDO = pSDO; if ( m_pSDO ) m_pSDO->AddRef(); }
+	~CSDORef() { if ( m_pSDO ) m_pSDO->Release(); }
+
+	T *Get() { return m_pSDO; }
+	const T *Get() const { return m_pSDO; }
+
+	T *operator->() { return Get(); }
+	const T *operator->() const { return Get(); }
+
+	operator const T *() const	{ return m_pSDO; }
+	operator const T *()        { return m_pSDO; }
+	operator T *()				{ return m_pSDO; }
+
+	CSDORef<T> &operator=( T *pSDO ) { if ( m_pSDO ) m_pSDO->Release();  m_pSDO = pSDO; if ( m_pSDO ) m_pSDO->AddRef(); return *this; }
+
+	bool operator !() const { return Get() == NULL; }
+
+	bool IsValid( void ) const { return Get() != NULL; }
+};
+
+
+//-----------------------------------------------------------------------------
+// Purpose: manages a cache of SDO objects
+//-----------------------------------------------------------------------------
+class CSDOCache
+{
+public:
+	CSDOCache();
+	~CSDOCache();
+
+	// Call to register SDOs. All SDO types must be registered before loaded
+	void RegisterSDO( int nType, const char *pchName );
+
+	// A struct to hold stats for the system. This is generated code in Steam. It would be great to make
+	// it generated code here if we could bring Steam's operational stats system in the GC
+	struct StatsSDOCache_t
+	{
+		uint64 m_cItemsLRUd;
+		uint64 m_cBytesLRUd;
+		uint64 m_cItemsUnreferenced;
+		uint64 m_cBytesUnreferenced;
+		uint64 m_cItemsInCache;
+		uint64 m_cBytesInCacheEst;
+		uint64 m_cItemsQueuedToLoad;
+		uint64 m_cItemsLoadedFromMemcached;
+		uint64 m_cItemsLoadedFromSQL;
+		uint64 m_cItemsFailedLoadFromSQL;
+		uint64 m_cQueuedMemcachedRequests;
+		uint64 m_cQueuedSQLRequests;
+		uint64 m_nSQLBatchSizeAvgx100;
+		uint64 m_nMemcachedBatchSizeAvgx100;
+		uint64 m_cSQLRequestsRejectedTooBusy;
+		uint64 m_cMemcachedRequestsRejectedTooBusy;
+		uint64 m_cNilItemsLoadedFromMemcached;
+		uint64 m_cNilItemsLoadedFromSQL;
+	};
+
+	// loads a SDO, and assigns a reference to it
+	// returns false if the item couldn't be loaded, or timed out loading
+	template<class T>
+	bool BYldLoadSDO( CSDORef<T> *pPSDORef, const typename T::KeyType_t &key, bool *pbTimeoutLoading = NULL );
+
+	// gets access to a SDO, but only if it's currently loaded
+	template<class T>
+	bool BGetLoadedSDO( CSDORef<T> *pPSDORef, const typename T::KeyType_t &key, bool *pbFoundNil = NULL );
+
+	// starts loading a SDO you're going to reference soon with the above BYldLoadSDO()
+	// use this to batch up requests, hinting a set then getting reference to a set is significantly faster
+	template<class T>
+	void HintLoadSDO( const typename T::KeyType_t &key );
+	// as above, but starts load a set
+	template<class T>
+	void HintLoadSDO( const CUtlVector<typename T::KeyType_t> &vecKeys );
+
+	// Clears a nil object if one exists for this key.
+	template<class T>
+	void RemoveNil( const typename T::KeyType_t &key );
+
+	// force a deletes a SDO from the cache - waits until the object is not referenced
+	template<class T>
+	bool BYldDeleteSDO( const typename T::KeyType_t &key, uint64 unMicrosecondsToWaitForUnreferenced );
+
+	// SDO refcount management
+	void OnSDOReferenced( ISDO *pSDO );
+	void OnSDOReleased( ISDO *pSDO );
+
+	// writes a SDO to memcached immediately
+	bool WriteSDOToMemcached( ISDO *pSDO );
+	// delete the SDO record from memcached
+	bool DeleteSDOFromMemcached( ISDO *pSDO );
+
+	// job results
+	void OnSDOLoadSuccess( int eSDO, int iRequestID, bool bNilObj, ISDO **ppSDO );
+	void OnMemcachedSDOLoadFailure( int eSDO, int iRequestID );
+	void OnSQLSDOLoadFailure( int eSDO, int iRequestID, bool bSQLLayerSucceeded );
+	void OnMemcachedLoadJobComplete( JobID_t jobID );
+	void OnSQLLoadJobComplete( int eSDO, JobID_t jobID );
+
+	// test access - deletes all unreferenced objects
+	void Flush();
+
+	// stats access
+	StatsSDOCache_t &GetStats()	{ return m_StatsSDOCache; }
+	int CubReferencedEst();	// number of bytes referenced in the cache
+
+	// prints info about the class
+	void Dump();
+
+	static void CreateSimpleMemcachedName( CFmtStr &strDest, const char *pchPrefix, uint32 unPrefixLen, uint32 unSuffix );
+
+	// memcached verification - returns the number of mismatches
+//**tempcomment**		void YldVerifyMemcachedData( CreateSDOFunc_t pCreateSDOFunc, CUtlVector<uint32> &vecIDs, int *pcMatches, int *pcMismatches );
+
+#ifdef DBGFLAG_VALIDATE
+	void Validate( CValidator &validator, const char *pchName );
+#endif
+
+	// Functions that need to be in the frame loop
+	virtual bool BFrameFuncRunJobsUntilCompleted( CLimitTimer &limitTimer );
+	virtual bool BFrameFuncRunMemcachedQueriesUntilCompleted( CLimitTimer &limitTimer );
+	virtual bool BFrameFuncRunSQLQueriesUntilCompleted( CLimitTimer &limitTimer );
+
+private:
+	// Custom comparator for our hash map
+	class CDefPISDOEquals
+	{
+	public:
+		CDefPISDOEquals() {}
+		CDefPISDOEquals( int i ) {}
+		inline bool operator()( const ISDO *lhs, const ISDO *rhs ) const { return ( lhs->IsEqual( rhs ) );	}
+		inline bool operator!() const { return false; }
+	};
+
+	class CPISDOHashFunctor
+	{
+	public:
+		uint32 operator()(const ISDO *pSDO ) const { return pSDO->GetHashCode(); }
+	};
+
+	template<class T>
+	int FindLoadedSDO( const typename T::KeyType_t &key );
+
+	template<class T>
+	int QueueLoad( const typename T::KeyType_t &key );
+	int QueueMemcachedLoad( ISDO *pSDO );
+
+	// items already loaded - Maps the SDO to the LRU position
+	CUtlHashMapLarge<ISDO *, int, CDefPISDOEquals, CPISDOHashFunctor> m_mapISDOLoaded;
+
+	// items we have queued to load, in the state of either being loaded from memcached or SQL
+	// maps SDO to a list of jobs waiting on the load
+	CUtlHashMapLarge<ISDO *, CCopyableUtlVector<JobID_t>, CDefPISDOEquals, CPISDOHashFunctor> m_mapQueuedRequests;
+
+	// requests to load from memcached
+	CUtlLinkedList<int, int> m_queueMemcachedRequests;
+
+	// Jobs currently processing memcached load requests
+	CUtlVector<JobID_t> m_vecMemcachedJobs;
+
+	// Loading from SQL is divided by SDO type
+	struct SQLRequestManager_t
+	{
+		// requests to load from SQL. Maps to an ID in the map of queued requests
+		CUtlLinkedList<int, int> m_queueRequestIDsToLoadFromSQL;
+
+		// SQL jobs we have active doing reads for cache items
+		CUtlVector<JobID_t> m_vecSQLJobs;
+	};
+
+	// a queue of requests to load from SQL for each type
+	CUtlHashMapLarge<int, SQLRequestManager_t *> m_mapQueueSQLRequests;
+
+	// jobs to wake up, since we've satisfied their SDO load request
+	struct JobToWake_t
+	{
+		JobID_t m_jobID;
+		bool m_bLoadLayerSuccess;
+	};
+	CUtlLinkedList<JobToWake_t, int> m_queueJobsToContinue;
+
+	struct LRUItem_t
+	{
+		ISDO *	m_pSDO;
+		size_t	m_cub;
+	};
+	CUtlLinkedList<LRUItem_t, int> m_listLRU;
+	uint32 m_cubLRUItems;
+	void RemoveSDOFromLRU( int iMapSDOLoaded );
+
+	struct TypeStats_t
+	{
+		TypeStats_t() 
+			: m_nLoaded( 0 )
+			, m_nRefed( 0 )
+			, m_cubUnrefed( 0 )
+			, m_nNilObjects( 0 )
+		{}
+
+		CUtlString	m_strName;
+		int			m_nLoaded;
+		int			m_nRefed;
+		int			m_cubUnrefed;
+		int			m_nNilObjects;
+	};
+
+	StatsSDOCache_t m_StatsSDOCache;
+	CMovingAverage<100> m_StatMemcachedBatchSize, m_StatSQLBatchSize;
+	CUtlMap<int, TypeStats_t> m_mapTypeStats;
+};
+
+
+//-----------------------------------------------------------------------------
+// Definition of CBaseSDO template functions now that CSDOCache is defined and
+// GSDOCache() can safely be used.
+//-----------------------------------------------------------------------------
+
+//-----------------------------------------------------------------------------
+// Purpose: adds a reference
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+int CBaseSDO<KeyType,ESDOType,ProtoMsg>::AddRef()
+{
+	if ( ++m_nRefCount == 1 )
+		GSDOCache().OnSDOReferenced( this );
+
+	return m_nRefCount;
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: releases a reference
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+int CBaseSDO<KeyType,ESDOType,ProtoMsg>::Release()
+{
+	DbgVerify( m_nRefCount > 0 );
+
+	int nRefCount = --m_nRefCount;
+
+	if ( nRefCount == 0 )
+		GSDOCache().OnSDOReleased( this );
+
+	return nRefCount;
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: ref count
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+int CBaseSDO<KeyType,ESDOType,ProtoMsg>::GetRefCount()
+{
+	return m_nRefCount;
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Hashes the object for insertion into a hashtable
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+uint32 CBaseSDO<KeyType,ESDOType,ProtoMsg>::GetHashCode() const
+{
+#pragma pack( push, 1 )
+	struct hashcode_t
+	{
+		int m_Type;
+		KeyType_t m_Key;
+	} hashStruct = { ESDOType, m_Key };
+#pragma pack( pop )
+
+	return PearsonsHashFunctor<hashcode_t>()( hashStruct );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Deserializes the object
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+bool CBaseSDO<KeyType,ESDOType,ProtoMsg>::BReadFromBuffer( const byte *pubData, int cubData )
+{
+	ProtoMsg msg;
+	if ( !msg.ParseFromArray( pubData, cubData ) )
+		return false;
+
+	if ( !DeserializeFromProtobuf( msg ) )
+		return false;
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Serializes the object
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+void CBaseSDO<KeyType,ESDOType,ProtoMsg>::WriteToBuffer( CUtlBuffer &memBuffer )
+{
+	ProtoMsg *pMsg = CProtoBufMsg<ProtoMsg>::AllocProto();
+
+	SerializeToProtobuf( *pMsg );
+	uint32 unSize = pMsg->ByteSize();
+	memBuffer.EnsureCapacity( memBuffer.Size() + unSize );
+	pMsg->SerializeWithCachedSizesToArray( (uint8*)memBuffer.Base() + memBuffer.TellPut() );
+	memBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, memBuffer.TellPut() + unSize );
+	
+	CProtoBufMsg<ProtoMsg>::FreeProto( pMsg );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: does an immediate write of the object to memcached
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+bool CBaseSDO<KeyType,ESDOType,ProtoMsg>::WriteToMemcached()
+{
+	return GSDOCache().WriteSDOToMemcached( this );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: does an immediate write of the object to memcached
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+bool CBaseSDO<KeyType,ESDOType,ProtoMsg>::DeleteFromMemcached()
+{
+	return GSDOCache().DeleteSDOFromMemcached( this );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: default equality function - compares type and key
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+bool CBaseSDO<KeyType,ESDOType,ProtoMsg>::IsEqual( const ISDO *pSDO ) const
+{
+	if ( GetType() != pSDO->GetType() )
+		return false;
+
+	return ( GetKey() == static_cast<const CBaseSDO<KeyType,ESDOType,ProtoMsg> *>( pSDO )->GetKey() );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Batch load a group of SDO's of the same type from SQL. 
+//  Default is no-op as not all types have permanent storage.
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+bool CBaseSDO<KeyType,ESDOType,ProtoMsg>::BYldLoadFromSQL( CUtlVector<ISDO *> &vecSDOToLoad, CUtlVector<bool> &vecResults ) const
+{
+	FOR_EACH_VEC( vecResults, i )
+	{
+		vecResults[i] = true;
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: default validation function - compares serialized versions
+//-----------------------------------------------------------------------------
+bool CompareSDOObjects( ISDO *pSDO1, ISDO *pSDO2 );
+
+template<typename KeyType, int ESDOType, class ProtoMsg>
+bool CBaseSDO<KeyType,ESDOType,ProtoMsg>::IsIdentical( ISDO *pSDO )
+{
+	return CompareSDOObjects( this, pSDO );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: default validation function - compares serialized versions
+//-----------------------------------------------------------------------------
+template<typename KeyType, int ESDOType, class ProtoMsg>
+ISDO *CBaseSDO<KeyType,ESDOType,ProtoMsg>::AllocNilObject()
+{
+	CFmtStr strKey;
+	GetMemcachedKeyName( strKey );
+	return new CNilSDO<KeyType,ESDOType,ProtoMsg>( GetKey(), strKey );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Finds a loaded SDO in memory. Returns the index of the object
+//	into the loaded SDOs map
+//-----------------------------------------------------------------------------
+template<class T>
+int CSDOCache::FindLoadedSDO( const typename T::KeyType_t &key )
+{
+	// see if we have it in cache first
+	T probe( key );
+	return m_mapISDOLoaded.Find( &probe );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Queues loading an SDO. Returns the index of the entry in the
+//	load queue
+//-----------------------------------------------------------------------------
+template<class T>
+int CSDOCache::QueueLoad( const typename T::KeyType_t &key )
+{
+	T probe( key );
+	int iMap = m_mapQueuedRequests.Find( &probe );
+	if ( m_mapQueuedRequests.IsValidIndex( iMap ) )
+		return iMap;
+
+	return QueueMemcachedLoad( new T( key ) );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Preloads the object into the local cache
+//-----------------------------------------------------------------------------
+template<class T>
+void CSDOCache::HintLoadSDO( const typename T::KeyType_t &key )
+{
+	// see if this is something we should even try to load
+	if ( !T::BKeyValid( key ) )
+		return;
+
+	// see if we have it in cache first
+	if ( !m_mapISDOLoaded.IsValidIndex( FindLoadedSDO<T>( key ) ) )
+	{
+		QueueLoad<T>( key );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Preloads a set set of objects into the local cache
+//-----------------------------------------------------------------------------
+template<class T>
+void CSDOCache::HintLoadSDO( const CUtlVector<typename T::KeyType_t> &vecKeys )
+{
+	FOR_EACH_VEC( vecKeys, i )
+	{
+		HintLoadSDO<T>( vecKeys[i] );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Returns an already-loaded SDO
+//-----------------------------------------------------------------------------
+template<class T>
+bool CSDOCache::BGetLoadedSDO( CSDORef<T> *pPSDORef, const typename T::KeyType_t &key, bool *pbFoundNil )
+{
+	if ( NULL != pbFoundNil )
+	{
+		*pbFoundNil = false;
+	}
+
+	int iMap = FindLoadedSDO<T>( key );
+	if ( !m_mapISDOLoaded.IsValidIndex( iMap ) )
+		return false;
+
+	ISDO *pObj = m_mapISDOLoaded.Key( iMap );
+	if ( pObj->BNilObject() )
+	{
+		int iLRU = m_mapISDOLoaded[ iMap ];
+		Assert( m_listLRU.IsValidIndex( iLRU ) );
+		if ( m_listLRU.IsValidIndex( iLRU ) )
+		{
+			// Even though we don't return nil objects, this is a hit on it
+			// so it needs to go to the back of the LRU
+			m_listLRU.LinkToTail( m_mapISDOLoaded[ iMap ] );
+		}
+
+		if ( NULL != pbFoundNil )
+		{
+			*pbFoundNil = true;
+		}
+
+		return false;
+	}
+	else
+	{
+		*pPSDORef = assert_cast<T*>( pObj );
+		return true;
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Loads the object into memory
+//-----------------------------------------------------------------------------
+template<class T>
+bool CSDOCache::BYldLoadSDO( CSDORef<T> *pPSDORef, const typename T::KeyType_t &key, bool *pbTimeoutLoading /* = NULL */ )
+{
+	VPROF_BUDGET( "CSDOCache::BYldLoadSDO", VPROF_BUDGETGROUP_STEAM );
+	if ( pbTimeoutLoading )
+		*pbTimeoutLoading = false;
+
+	// Clear the current object the ref is holding
+	*pPSDORef = NULL;
+
+	// see if this is something we should even try to load
+	if ( !T::BKeyValid( key ) )
+		return false;
+
+	// see if we have it in cache first
+	bool bFoundNil = false;
+	if ( BGetLoadedSDO( pPSDORef, key, &bFoundNil ) )
+		return true;
+
+	// If we've already tried to look this up in the past don't bother looking it up again
+	if ( bFoundNil )
+		return false;
+
+	// otherwise batch it for load
+	int iMap = QueueLoad<T>( key );
+
+	// make sure we could queue it
+	if ( !m_mapQueuedRequests.IsValidIndex( iMap ) )
+		return false;
+
+	// add the current job to this list waiting for the object to load 
+	m_mapQueuedRequests[iMap].AddToTail( GJobCur().GetJobID() );
+
+	// wait for it to load (loader will signal our job when done)
+	if ( !GJobCur().BYieldingWaitForWorkItem() )
+	{
+		if ( pbTimeoutLoading )
+			*pbTimeoutLoading = true;
+		return false;
+	}
+
+	// should be loaded - look up in the load map and try again
+	bool bRet = BGetLoadedSDO( pPSDORef, key );
+	//Assert( bRet );
+
+	return bRet;
+}
+
+
+//-----------------------------------------------------------------------------
+// Clears a nil object if one exists for this key.
+//-----------------------------------------------------------------------------
+template<class T>
+void CSDOCache::RemoveNil( const typename T::KeyType_t &key )
+{
+	// See if this key is allowed to exist
+	if ( !T::BKeyValid( key ) )
+	{
+		AssertMsg( false, "RemoveNil called with an invalid key" );
+		return;
+	}
+
+	// Remove the nil if there is one
+	int iMap = FindLoadedSDO< T >( key );
+	if ( m_mapISDOLoaded.IsValidIndex( iMap ) )
+	{
+		ISDO *pSDO = m_mapISDOLoaded.Key( iMap );
+		if ( !pSDO->BNilObject() )
+			return;
+
+		int iMapStats = m_mapTypeStats.Find( pSDO->GetType() );
+		if ( m_mapTypeStats.IsValidIndex( iMapStats ) )
+		{
+			m_mapTypeStats[iMapStats].m_nNilObjects--;
+		}
+
+		RemoveSDOFromLRU( iMap );
+		m_mapISDOLoaded.RemoveAt( iMap );
+		delete pSDO;
+	}
+
+	// Remove the object from memcached. Do this here because while we have for sure removed any nil that was in memory
+	// this may have been a nil that was not in memory but instead cached in memcached.
+	T temp( key );
+	DeleteSDOFromMemcached( &temp );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: reloads an existing element from the SQL DB
+//-----------------------------------------------------------------------------
+template<class T>
+bool CSDOCache::BYldDeleteSDO( const typename T::KeyType_t &key, uint64 unMicrosecondsToWaitForUnreferenced )
+{
+	// see if we have it in cache first
+	int iMap = FindLoadedSDO<T>( key );
+	if ( !m_mapISDOLoaded.IsValidIndex( iMap ) )
+	{
+		T temp( key );
+		temp.DeleteFromMemcached();
+		return true; /* we're good, it's not loaded */
+	}
+
+	assert_cast<T *>(m_mapISDOLoaded.Key( iMap ))->DeleteFromMemcached();
+
+	// check the ref count
+	int64 cAttempts = MAX( 1LL, (int64)(unMicrosecondsToWaitForUnreferenced / k_cMicroSecPerShellFrame) );
+	while ( cAttempts-- > 0 )
+	{
+		if ( 0 == m_mapISDOLoaded.Key( iMap )->GetRefCount() )
+		{
+			// delete the object
+			Assert( m_listLRU.IsValidIndex( m_mapISDOLoaded[iMap] ) );
+
+			RemoveSDOFromLRU( iMap );
+			ISDO *pSDO = m_mapISDOLoaded.Key( iMap );
+			m_mapISDOLoaded.RemoveAt( iMap );
+
+			int iMapStats = m_mapTypeStats.Find( m_mapISDOLoaded.Key( iMap )->GetType() );
+			if ( m_mapTypeStats.IsValidIndex( iMapStats ) )
+			{
+				if ( pSDO->BNilObject() )
+				{
+					m_mapTypeStats[iMapStats].m_nNilObjects--;
+				}
+				else
+				{
+					m_mapTypeStats[iMapStats].m_nLoaded--;
+				}
+			}
+
+			delete pSDO;
+			return true;
+		}
+		else
+		{
+			GJobCur().BYieldingWaitOneFrame();
+		}
+	}
+	
+	// couldn't reload
+	return false;
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: A class to factor out the common code in most SDO SQL loading funcitons
+//-----------------------------------------------------------------------------
+template<class T>
+class CSDOSQLLoadHelper
+{
+public:
+
+	//this is the results of the load helper, abstracted to provide a more efficient means to lookup the queries without
+	//having to do very expensive memory management
+	template< class SCH >
+	class CResults
+	{
+	public:
+
+		//given a key, this will return the range of indices that include this key. -1 will be set to the start if no match is found. This will return
+		//the number of matches found
+		int			GetKeyIndexRange( typename T::KeyType_t Key, int& nStart, int& nEnd ) const;
+
+		//given a key, this will return the first schema that matches this key, or NULL if none is found. Only use this if you know there
+		//is a maximum of a single value
+		const SCH*	GetSingleResultForKey( typename T::KeyType_t Key ) const;
+
+		//used to start iteration over a group of results. Given a key, this will return the index that can be used to get the result or -1 if no match is found
+		int			GetFirstResultIndex( typename T::KeyType_t Key ) const;
+		//given an index that was previously obtained from GetFirst/NextResultIndex, this will get the next result that has the same key, or return -1 if no further keys of that type are found
+		int			GetNextResultIndex( int nOldResultIndex ) const;
+		//given an index from GetFirst/NextResultIndex that is valid (i.e. not -1), this will return the result that was obtained
+		const SCH*	GetResultFromIndex( int nIndex ) const;
+		//given an index returned by the GetFirst/NextResultIndex, this will determine if it is valid
+		static int	InvalidIndex()		{ return -1; }
+
+	private:
+		//given a key, this maps to the specific result
+		template< class TKeyType >
+		struct SKeyToResult
+		{
+			bool operator< (const SKeyToResult< TKeyType >& rhs ) const			{ return m_Key < rhs.m_Key; }
+			typename TKeyType	m_Key;				//key value
+			uint32				m_nResultIndex;		//index into our result list
+		};
+
+		friend class CSDOSQLLoadHelper;
+
+		CUtlVector< SKeyToResult< typename T::KeyType_t > >	m_KeyToResult;
+		CUtlVector< SCH >							m_Results;
+	};
+
+
+	// Initializes with the vector of objects being loaded
+	CSDOSQLLoadHelper( const CUtlVector<ISDO *> *vecSDOToLoad, const char *pchProfileName );
+
+	// Loads all rows in the SCH table whose field nFieldID match the key of an SDO being loaded
+	template<class SCH>
+	bool BYieldingExecuteSingleTable( int nFieldID, CResults< SCH >& Results );
+	// Loads the specified columns for all rows in the SCH table whose field nFieldID match the key of an SDO being loaded
+	template<class SCH>
+	bool BYieldingExecuteSingleTable( int nFieldID, const CColumnSet &csetRead, CResults< SCH >& Results );
+
+	// Functions to load rows from more than one table at a time
+
+	// Loads all rows in the SCH table whose field nFieldID match the key of an SDO being loaded
+	template<class SCH>
+	void AddTableToQuery( int nFieldID );
+
+	// Loads the specified columns for all rows in the SCH table whose field nFieldID match the key of an SDO being loaded
+	template<class SCH>
+	void AddTableToQuery( int nFieldID, const CColumnSet &csetRead );
+
+	// Executes the mutli-table query
+	bool BYieldingExecute();
+		
+	// Gets the results for a table from a multi-table query
+	template<class SCH>
+	bool BGetResults( int nQuery, CResults< SCH >& Results );
+
+private:
+	CUtlVector<typename T::KeyType_t> m_vecKeys;
+	CSQLAccess m_sqlAccess;
+
+	struct Query_t
+	{
+		Query_t( const CColumnSet &columnSet, int nKeyCol ) : m_ColumnSet( columnSet ), m_nKeyCol( nKeyCol ) {}
+		CColumnSet	m_ColumnSet;
+		int			m_nKeyCol;
+	};
+
+	CUtlVector<Query_t> m_vecQueries;
+};
+
+//utility to help with iteration through a SQL load result list
+#define FOR_EACH_SQL_LOAD_RESULT( Results, Key, Index )		for( int Index = Results.GetFirstResultIndex( Key ); Index != Results.InvalidIndex(); Index = Results.GetNextResultIndex( Index ) )
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Constructor. Initializes with the vector of objects being loaded
+//-----------------------------------------------------------------------------
+template<class T>
+CSDOSQLLoadHelper<T>::CSDOSQLLoadHelper( const CUtlVector<ISDO *> *vecSDOToLoad, const char *pchProfileName )
+	: m_vecKeys( 0, vecSDOToLoad->Count() )
+{
+	FOR_EACH_VEC( *vecSDOToLoad, i )
+	{
+		m_vecKeys.AddToTail( ( (T*)vecSDOToLoad->Element( i ) )->GetKey() );
+	}
+
+	Assert( m_vecKeys.Count() > 0 );
+	DbgVerify( m_sqlAccess.BBeginTransaction( pchProfileName ) );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Loads all rows in the SCH table whose field nFieldID match the 
+//	key of an SDO being loaded.
+//-----------------------------------------------------------------------------
+template<class T>
+template<class SCH>
+bool CSDOSQLLoadHelper<T>::BYieldingExecuteSingleTable( int nFieldID, CResults< SCH >& Results )
+{
+	static const CColumnSet cSetRead = CColumnSet::Full<SCH>();
+	return BYieldingExecuteSingleTable( nFieldID, cSetRead, Results );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Loads the specified columns for all rows in the SCH table whose 
+//	field nFieldID match the key of an SDO being loaded
+//-----------------------------------------------------------------------------
+template<class T>
+template<class SCH>
+bool CSDOSQLLoadHelper<T>::BYieldingExecuteSingleTable( int nFieldID, const CColumnSet &csetRead, CResults< SCH >& Results )
+{
+	AddTableToQuery<SCH>( nFieldID, csetRead );
+	if ( !BYieldingExecute() )
+		return false;
+
+	return BGetResults<SCH>( 0, Results );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Loads all rows in the SCH table whose field nFieldID match the key 
+//	of an SDO being loaded
+//-----------------------------------------------------------------------------
+template<class T>
+template<class SCH>
+void CSDOSQLLoadHelper<T>::AddTableToQuery( int nFieldID )
+{
+	static const CColumnSet cSetRead = CColumnSet::Full<SCH>();
+	AddTableToQuery<SCH>( nFieldID, cSetRead );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Loads the specified columns for all rows in the SCH table whose 
+//	field nFieldID match the key of an SDO being loaded
+//-----------------------------------------------------------------------------
+template<class T>
+template<class SCH>
+void CSDOSQLLoadHelper<T>::AddTableToQuery( int nFieldID, const CColumnSet &csetRead )
+{
+	Assert( csetRead.GetRecordInfo() == GSchemaFull().GetSchema( SCH::k_iTable ).GetRecordInfo() );
+
+	// Bind the params
+	FOR_EACH_VEC( m_vecKeys, i )
+	{
+		m_sqlAccess.AddBindParam( m_vecKeys[i] );
+	}
+
+	// Build the query
+	CUtlString sCommand;
+	{
+		TSQLCmdStr sSelect;
+		const char *pchColumnName = GSchemaFull().GetSchema( SCH::k_iTable ).GetRecordInfo()->GetColumnInfo( nFieldID ).GetName();
+
+		BuildSelectStatementText( &sSelect, csetRead );
+		sCommand.Format( "%s WHERE %s IN (%.*s)", sSelect.Access(), pchColumnName, GetInsertArgStringChars( m_vecKeys.Count() ), GetInsertArgString() );
+	}
+
+	// Execute. Because we're in a transaction this will delay to the commit
+	DbgVerify( m_sqlAccess.BYieldingExecute( NULL, sCommand ) );
+
+	m_vecQueries.AddToTail( Query_t( csetRead, nFieldID ) );
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Executes the mutli-table query
+//-----------------------------------------------------------------------------
+template<class T>
+bool CSDOSQLLoadHelper<T>::BYieldingExecute()
+{
+	if ( 0 == m_vecKeys.Count() )
+	{
+		m_sqlAccess.RollbackTransaction();
+		return false;
+	}
+
+	if ( !m_sqlAccess.BCommitTransaction() )
+		return false;
+
+	Assert( (uint32)m_vecQueries.Count() == m_sqlAccess.GetResultSetCount() );
+	return (uint32)m_vecQueries.Count() == m_sqlAccess.GetResultSetCount();
+}
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Gets the results for a table from a multi-table query
+//-----------------------------------------------------------------------------
+template<class T>
+template<class SCH>
+bool CSDOSQLLoadHelper<T>::BGetResults( int nQuery, CResults< SCH >& Results )
+{
+	//clear any previous results
+	Results.m_KeyToResult.Purge();
+	Results.m_Results.Purge();
+
+	IGCSQLResultSetList *pSQLResults = m_sqlAccess.GetResults();
+	Assert( pSQLResults && nQuery >= 0 && (uint32)nQuery < pSQLResults->GetResultSetCount() && pSQLResults->GetResultSetCount() == (uint32)m_vecQueries.Count() );
+	if ( NULL == pSQLResults || nQuery < 0 || (uint32)nQuery >= pSQLResults->GetResultSetCount() || pSQLResults->GetResultSetCount() != (uint32)m_vecQueries.Count() )
+		return false;
+
+	Assert( m_vecQueries[nQuery].m_ColumnSet.GetRecordInfo()->GetTableID() == SCH::k_iTable );
+	if ( m_vecQueries[nQuery].m_ColumnSet.GetRecordInfo()->GetTableID() != SCH::k_iTable )
+		return false;
+
+	//copy the results from the SQL queries over to our result list
+	Results.m_Results.EnsureCapacity( pSQLResults->GetResultSet( nQuery )->GetRowCount() );
+	if ( !CopyResultToSchVector( pSQLResults->GetResultSet( nQuery ), m_vecQueries[nQuery].m_ColumnSet, &Results.m_Results ) )
+		return false;
+
+	// Make a map that counts maps from our results into a sorted list for fast lookup
+	Results.m_KeyToResult.SetSize( Results.m_Results.Count() );
+	FOR_EACH_VEC( Results.m_Results, nCurrResult )
+	{
+		//get our key value
+		uint8 *pubData;
+		uint32 cubData;
+		if ( !Results.m_Results[ nCurrResult ].BGetField( m_vecQueries[nQuery].m_nKeyCol, &pubData, &cubData ) )
+			return false;
+
+		Assert( cubData == sizeof( T::KeyType_t ) );
+		if ( cubData != sizeof( T::KeyType_t ) )
+		{
+			Results.m_KeyToResult.Purge();
+			Results.m_Results.Purge();
+			return false;
+		}
+
+		//setup our record
+		Results.m_KeyToResult[ nCurrResult ].m_Key = *((T::KeyType_t *)pubData);
+		Results.m_KeyToResult[ nCurrResult ].m_nResultIndex = nCurrResult;
+	}
+
+	//sort for binary search capabilities
+	std::sort( Results.m_KeyToResult.begin(), Results.m_KeyToResult.end() );
+
+	return true;
+}
+
+template< typename T >
+template< typename SCH >
+int CSDOSQLLoadHelper< T >::CResults< SCH >::GetKeyIndexRange( typename T::KeyType_t Key, int& nStart, int& nEnd ) const
+{
+	//find the start of the range
+	nStart = GetFirstResultIndex( Key );
+	if( nStart == InvalidIndex() )
+	{
+		nEnd = InvalidIndex();
+		return 0;
+	}
+	//expand the end as long as it lies on a key in range and with the same value
+	for( nEnd = nStart + 1; ( nEnd < m_KeyToResult.Count() ) && ( m_KeyToResult[ nEnd ].m_Key == Key ); nEnd++ )
+	{
+	}
+	//and return the resulting number of elements we found
+	return nEnd - nStart;
+}
+
+
+template< typename T >
+template< typename SCH >
+const SCH* CSDOSQLLoadHelper< T >::CResults< SCH >::GetSingleResultForKey( typename T::KeyType_t Key ) const
+{
+	int nIndex = GetFirstResultIndex( Key );
+	AssertMsg( ( nIndex == InvalidIndex() ) || ( GetNextResultIndex( nIndex ) == InvalidIndex() ), "Requested a result from a SQL load helper assuming it was a singular entry, but found multiple instances of it" );
+	return GetResultFromIndex( nIndex );
+}
+
+template< typename T >
+template< typename SCH >
+int CSDOSQLLoadHelper< T >::CResults< SCH >::GetFirstResultIndex( typename T::KeyType_t Key ) const
+{
+	//dummy entry to compare against
+	SKeyToResult< typename T::KeyType_t > SearchKey;
+	SearchKey.m_Key = Key;	
+
+	//binary search to find our index
+	const SKeyToResult< typename T::KeyType_t >* pMatch = std::lower_bound( m_KeyToResult.begin(), m_KeyToResult.end(), SearchKey );
+	//see if we found a match
+	if( ( pMatch == m_KeyToResult.end() ) || ( pMatch->m_Key != Key ) )
+		return InvalidIndex();
+
+	return ( pMatch - m_KeyToResult.begin() );
+}
+
+template< typename T >
+template< typename SCH >
+int CSDOSQLLoadHelper< T >::CResults< SCH >::GetNextResultIndex( int nOldResultIndex ) const
+{
+	//handle out of range elements. Either invalid, or the last element or beyond in our array
+	if( ( nOldResultIndex < 0 ) || ( nOldResultIndex + 1 >= m_KeyToResult.Count() ) )
+		return InvalidIndex();
+
+	//see if we are less than the next value in the list. If so, we aren't equal and need to stop iteration	
+	if( m_KeyToResult[ nOldResultIndex ] < m_KeyToResult[ nOldResultIndex + 1 ] )
+		return InvalidIndex();
+	
+	//same key for the next element, so return a match
+	return nOldResultIndex + 1;
+}
+
+template< typename T >
+template< typename SCH >
+const SCH* CSDOSQLLoadHelper< T >::CResults< SCH >::GetResultFromIndex( int nIndex ) const
+{
+	//ensure that the index is in range
+	if( ( nIndex < 0 ) || ( nIndex >= m_KeyToResult.Count() ) )
+		return NULL;
+	return &( m_Results[ m_KeyToResult[ nIndex ].m_nResultIndex ] );
+}
+
+} // namespace GCSDK
+
+#endif // SDOCACHE_H